1999-01-30 01:19:00 +00:00
|
|
|
;;; The contents of this file are subject to the Netscape Public License
|
|
|
|
|
;;; Version 1.0 (the "NPL"); you may not use this file except in
|
|
|
|
|
;;; compliance with the NPL. You may obtain a copy of the NPL at
|
|
|
|
|
;;; http://www.mozilla.org/NPL/
|
|
|
|
|
;;;
|
|
|
|
|
;;; Software distributed under the NPL is distributed on an "AS IS" basis,
|
|
|
|
|
;;; WITHOUT WARRANTY OF ANY KIND, either express or implied. See the NPL
|
|
|
|
|
;;; for the specific language governing rights and limitations under the
|
|
|
|
|
;;; NPL.
|
|
|
|
|
;;;
|
|
|
|
|
;;; The Initial Developer of this code under the NPL is Netscape
|
|
|
|
|
;;; Communications Corporation. Portions created by Netscape are
|
|
|
|
|
;;; Copyright (C) 1998 Netscape Communications Corporation. All Rights
|
|
|
|
|
;;; Reserved.
|
|
|
|
|
|
|
|
|
|
;;;
|
|
|
|
|
;;; LALR(1) and LR(1) grammar generator
|
|
|
|
|
;;;
|
|
|
|
|
;;; Waldemar Horwat (waldemar@netscape.com)
|
|
|
|
|
;;;
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
;;; ------------------------------------------------------------------------------------------------------
|
|
|
|
|
|
|
|
|
|
; kernel-item-alist is a list of pairs (item . prev), where item is a kernel item
|
|
|
|
|
; and prev is either nil or a laitem. kernel is a list of the kernel items in a canonical order.
|
|
|
|
|
; Return a new state with the given list of kernel items and state number.
|
|
|
|
|
; For each non-null prev in kernel-item-alist, update (laitem-propagates prev) to include the
|
|
|
|
|
; corresponding laitem in the new state.
|
|
|
|
|
(defun make-state (grammar kernel kernel-item-alist number initial-lookaheads)
|
|
|
|
|
(let ((laitems nil)
|
|
|
|
|
(laitems-hash (make-hash-table :test #'eq)))
|
|
|
|
|
(labels
|
|
|
|
|
;Create a laitem for this item and add the association item->laitem to the laitems-hash
|
|
|
|
|
;hash table if it's not there already. Regardless of whether a new laitem was created,
|
|
|
|
|
;update the laitem's lookaheads to also include the given lookaheads.
|
|
|
|
|
;If prev is non-null, update (laitem-propagates prev) to include the laitem if it's not
|
|
|
|
|
;already included there.
|
|
|
|
|
;If a new laitem was created and its first symbol after the dot exists and is a
|
|
|
|
|
;nonterminal A, recursively close items A->.rhs corresponding to all rhs's in the
|
|
|
|
|
;grammar's rule for A.
|
|
|
|
|
((close-item (item lookaheads prev)
|
|
|
|
|
(let ((laitem (gethash item laitems-hash)))
|
|
|
|
|
(if laitem
|
|
|
|
|
(setf (laitem-lookaheads laitem)
|
|
|
|
|
(terminalset-union (laitem-lookaheads laitem) lookaheads))
|
|
|
|
|
(let ((item-next-symbol (item-next-symbol item)))
|
|
|
|
|
(setq laitem (allocate-laitem grammar item lookaheads))
|
|
|
|
|
(push laitem laitems)
|
|
|
|
|
(setf (gethash item laitems-hash) laitem)
|
|
|
|
|
(when (nonterminal? item-next-symbol)
|
|
|
|
|
(multiple-value-bind (next-lookaheads epsilon-lookahead)
|
|
|
|
|
(string-initial-terminals grammar (rest (item-unseen item)))
|
|
|
|
|
(let ((next-prev (and epsilon-lookahead laitem)))
|
|
|
|
|
(dolist (production (rule-productions (grammar-rule grammar item-next-symbol)))
|
|
|
|
|
(close-item (make-item grammar production 0) next-lookaheads next-prev)))))))
|
|
|
|
|
(when prev
|
|
|
|
|
(pushnew laitem (laitem-propagates prev))))))
|
|
|
|
|
|
|
|
|
|
(dolist (acons kernel-item-alist nil)
|
|
|
|
|
(let ((item (car acons))
|
|
|
|
|
(prev (cdr acons)))
|
|
|
|
|
(close-item item initial-lookaheads prev)))
|
|
|
|
|
(allocate-state number kernel (nreverse laitems)))))
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
; f is a function that takes two arguments:
|
|
|
|
|
; a grammar symbol, and
|
|
|
|
|
; a list of kernel items in order of increasing item number.
|
|
|
|
|
; a list of pairs (item . prev), where item is a kernel item and prev is a laitem;
|
|
|
|
|
; For each possible symbol X that can be shifted while in the given state S, call
|
|
|
|
|
; f giving it S and the list of items that constitute the kernel of that shift's destination
|
|
|
|
|
; state. The prev's are the sources of the corresponding shifted items.
|
|
|
|
|
(defun state-each-shift-item-alist (f state)
|
|
|
|
|
(let ((shift-symbols-hash (make-hash-table :test *grammar-symbol-=*)))
|
|
|
|
|
(dolist (source-laitem (state-laitems state))
|
|
|
|
|
(let* ((source-item (laitem-item source-laitem))
|
|
|
|
|
(shift-symbol (item-next-symbol source-item)))
|
|
|
|
|
(when shift-symbol
|
|
|
|
|
(push (cons (item-next source-item) source-laitem)
|
|
|
|
|
(gethash shift-symbol shift-symbols-hash)))))
|
|
|
|
|
;Use dolist/gethash instead of maphash to make state assignments deterministic.
|
|
|
|
|
(dolist (shift-symbol (sorted-hash-table-keys shift-symbols-hash))
|
|
|
|
|
(let ((kernel-item-alist (gethash shift-symbol shift-symbols-hash)))
|
|
|
|
|
(funcall f shift-symbol (sort (mapcar #'car kernel-item-alist) #'< :key #'item-number) kernel-item-alist)))))
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
;;; ------------------------------------------------------------------------------------------------------
|
|
|
|
|
;;; LR(1)
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
; kernel-item-alist should have the same kernel items as state.
|
|
|
|
|
; Return true if the prev lookaheads in kernel-item-alist are the same as or subsets of
|
|
|
|
|
; the corresponding lookaheads in the state's kernel laitems.
|
|
|
|
|
(defun state-subsumes-lookaheads (state kernel-item-alist)
|
|
|
|
|
(every
|
|
|
|
|
#'(lambda (acons)
|
|
|
|
|
(terminalset-<= (laitem-lookaheads (cdr acons))
|
|
|
|
|
(laitem-lookaheads (state-laitem state (car acons)))))
|
|
|
|
|
kernel-item-alist))
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
; kernel-item-alist should have the same kernel items as state.
|
|
|
|
|
; Return true if the prev lookaheads in kernel-item-alist are weakly compatible
|
|
|
|
|
; with the lookaheads in the state's kernel laitems.
|
|
|
|
|
(defun state-weakly-compatible (state kernel-item-alist)
|
|
|
|
|
(labels
|
|
|
|
|
((lookahead-weakly-compatible (lookahead1a lookahead1b lookahead2a lookahead2b)
|
|
|
|
|
(or (and (terminalsets-disjoint lookahead1a lookahead2b)
|
|
|
|
|
(terminalsets-disjoint lookahead1b lookahead2a))
|
|
|
|
|
(not (terminalsets-disjoint lookahead1a lookahead1b))
|
|
|
|
|
(not (terminalsets-disjoint lookahead2a lookahead2b))))
|
|
|
|
|
|
|
|
|
|
(lookahead-list-weakly-compatible (lookahead1a lookaheads1 lookahead2a lookaheads2)
|
|
|
|
|
(or (endp lookaheads1)
|
|
|
|
|
(and (lookahead-weakly-compatible lookahead1a (first lookaheads1) lookahead2a (first lookaheads2))
|
|
|
|
|
(lookahead-list-weakly-compatible lookahead1a (rest lookaheads1) lookahead2a (rest lookaheads2)))))
|
|
|
|
|
|
|
|
|
|
(lookahead-lists-weakly-compatible (lookaheads1 lookaheads2)
|
|
|
|
|
(or (endp lookaheads1)
|
|
|
|
|
(and (lookahead-list-weakly-compatible (first lookaheads1) (rest lookaheads1) (first lookaheads2) (rest lookaheads2))
|
|
|
|
|
(lookahead-lists-weakly-compatible (rest lookaheads1) (rest lookaheads2))))))
|
|
|
|
|
|
|
|
|
|
(or (= (length kernel-item-alist) 1)
|
|
|
|
|
(lookahead-lists-weakly-compatible
|
|
|
|
|
(mapcar #'(lambda (acons) (laitem-lookaheads (state-laitem state (car acons)))) kernel-item-alist)
|
|
|
|
|
(mapcar #'(lambda (acons) (laitem-lookaheads (cdr acons))) kernel-item-alist)))))
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
; Propagate all lookaheads in the state.
|
|
|
|
|
(defun propagate-internal-lookaheads (state)
|
|
|
|
|
(dolist (src-laitem (state-laitems state))
|
|
|
|
|
(let ((src-lookaheads (laitem-lookaheads src-laitem)))
|
|
|
|
|
(dolist (dst-laitem (laitem-propagates src-laitem))
|
|
|
|
|
(setf (laitem-lookaheads dst-laitem)
|
|
|
|
|
(terminalset-union (laitem-lookaheads dst-laitem) src-lookaheads))))))
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
; Propagate all lookaheads in kernel-item-alist, which must target destination-state.
|
|
|
|
|
; Mark destination-state as dirty in the dirty-states hash table.
|
|
|
|
|
(defun propagate-external-lookaheads (kernel-item-alist destination-state dirty-states)
|
|
|
|
|
(dolist (acons kernel-item-alist)
|
|
|
|
|
(let ((dest-laitem (state-laitem destination-state (car acons)))
|
|
|
|
|
(src-laitem (cdr acons)))
|
|
|
|
|
(setf (laitem-lookaheads dest-laitem)
|
|
|
|
|
(terminalset-union (laitem-lookaheads dest-laitem) (laitem-lookaheads src-laitem)))))
|
|
|
|
|
(setf (gethash destination-state dirty-states) t))
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
; Make all states in the grammar and return the initial state.
|
|
|
|
|
; Initialize the grammar's list of states.
|
|
|
|
|
; Set up the laitems' propagate lists but do not propagate lookaheads yet.
|
|
|
|
|
; Initialize the states' gotos lists.
|
|
|
|
|
; Initialize the states' shift (but not reduce or accept) transitions in the transitions lists.
|
|
|
|
|
(defun add-all-lr-states (grammar)
|
|
|
|
|
(let* ((initial-item (make-item grammar (grammar-start-production grammar) 0))
|
|
|
|
|
(lr-states-hash (make-hash-table :test #'equal)) ;kernel -> list of states with that kernel
|
|
|
|
|
(initial-kernel (list initial-item))
|
|
|
|
|
(initial-state (make-state grammar initial-kernel (list (cons initial-item nil)) 0 (make-terminalset grammar *end-marker*)))
|
|
|
|
|
(states (list initial-state))
|
|
|
|
|
(next-state-number 1))
|
|
|
|
|
(setf (gethash initial-kernel lr-states-hash) (list initial-state))
|
|
|
|
|
(do ((source-states (list initial-state))
|
|
|
|
|
(dirty-states (make-hash-table :test #'eq))) ;Set of states whose kernel lookaheads changed and haven't been propagated yet
|
|
|
|
|
((and (endp source-states) (zerop (hash-table-count dirty-states))))
|
|
|
|
|
(labels
|
|
|
|
|
((make-destination-state (kernel kernel-item-alist)
|
|
|
|
|
(let* ((possible-destination-states (gethash kernel lr-states-hash))
|
|
|
|
|
(destination-state (find-if #'(lambda (possible-destination-state)
|
|
|
|
|
(state-subsumes-lookaheads possible-destination-state kernel-item-alist))
|
|
|
|
|
possible-destination-states)))
|
|
|
|
|
(cond
|
|
|
|
|
(destination-state)
|
|
|
|
|
((setq destination-state (find-if #'(lambda (possible-destination-state)
|
|
|
|
|
(state-weakly-compatible possible-destination-state kernel-item-alist))
|
|
|
|
|
possible-destination-states))
|
|
|
|
|
(propagate-external-lookaheads kernel-item-alist destination-state dirty-states))
|
|
|
|
|
(t
|
|
|
|
|
(setq destination-state (make-state grammar kernel kernel-item-alist next-state-number *empty-terminalset*))
|
|
|
|
|
(propagate-external-lookaheads kernel-item-alist destination-state dirty-states)
|
|
|
|
|
(push destination-state (gethash kernel lr-states-hash))
|
|
|
|
|
(incf next-state-number)
|
|
|
|
|
(push destination-state states)
|
|
|
|
|
(push destination-state source-states)))
|
|
|
|
|
destination-state))
|
|
|
|
|
|
|
|
|
|
(update-destination-state (destination-state kernel-item-alist)
|
|
|
|
|
(cond
|
|
|
|
|
((state-subsumes-lookaheads destination-state kernel-item-alist)
|
|
|
|
|
destination-state)
|
|
|
|
|
((state-weakly-compatible destination-state kernel-item-alist)
|
|
|
|
|
(propagate-external-lookaheads kernel-item-alist destination-state dirty-states)
|
|
|
|
|
destination-state)
|
|
|
|
|
(t (make-destination-state (state-kernel destination-state) kernel-item-alist)))))
|
|
|
|
|
|
|
|
|
|
(if source-states
|
|
|
|
|
(let ((source-state (pop source-states)))
|
|
|
|
|
(remhash source-state dirty-states)
|
|
|
|
|
(propagate-internal-lookaheads source-state)
|
|
|
|
|
(state-each-shift-item-alist
|
|
|
|
|
#'(lambda (shift-symbol kernel kernel-item-alist)
|
|
|
|
|
(let ((destination-state (make-destination-state kernel kernel-item-alist)))
|
|
|
|
|
(if (nonterminal? shift-symbol)
|
|
|
|
|
(push (cons shift-symbol destination-state)
|
|
|
|
|
(state-gotos source-state))
|
|
|
|
|
(push (cons shift-symbol (make-shift-transition destination-state))
|
|
|
|
|
(state-transitions source-state)))))
|
|
|
|
|
source-state))
|
|
|
|
|
(dolist (dirty-state (sort (hash-table-keys dirty-states) #'< :key #'state-number))
|
|
|
|
|
(when (remhash dirty-state dirty-states)
|
|
|
|
|
(propagate-internal-lookaheads dirty-state)
|
|
|
|
|
(state-each-shift-item-alist
|
|
|
|
|
#'(lambda (shift-symbol kernel kernel-item-alist)
|
|
|
|
|
(declare (ignore kernel))
|
|
|
|
|
(if (nonterminal? shift-symbol)
|
|
|
|
|
(let* ((destination-binding (assoc shift-symbol (state-gotos dirty-state) :test *grammar-symbol-=*))
|
|
|
|
|
(destination-state (assert-non-null (cdr destination-binding))))
|
|
|
|
|
(setf (cdr destination-binding) (update-destination-state destination-state kernel-item-alist)))
|
|
|
|
|
(let* ((destination-transition (cdr (assoc shift-symbol (state-transitions dirty-state) :test *grammar-symbol-=*)))
|
|
|
|
|
(destination-state (assert-non-null (transition-state destination-transition))))
|
|
|
|
|
(setf (transition-state destination-transition)
|
|
|
|
|
(update-destination-state destination-state kernel-item-alist)))))
|
|
|
|
|
dirty-state))))))
|
|
|
|
|
(setf (grammar-states grammar) (nreverse states))
|
|
|
|
|
initial-state))
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
;;; ------------------------------------------------------------------------------------------------------
|
|
|
|
|
;;; LALR(1)
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
; Make all states in the grammar and return the initial state.
|
|
|
|
|
; Initialize the grammar's list of states.
|
|
|
|
|
; Set up the laitems' propagate lists but do not propagate lookaheads yet.
|
|
|
|
|
; Initialize the states' gotos lists.
|
|
|
|
|
; Initialize the states' shift (but not reduce or accept) transitions in the transitions lists.
|
|
|
|
|
(defun add-all-lalr-states (grammar)
|
|
|
|
|
(let* ((initial-item (make-item grammar (grammar-start-production grammar) 0))
|
|
|
|
|
(lalr-states-hash (make-hash-table :test #'equal)) ;kernel -> state
|
|
|
|
|
(initial-kernel (list initial-item))
|
|
|
|
|
(initial-state (make-state grammar initial-kernel (list (cons initial-item nil)) 0 (make-terminalset grammar *end-marker*)))
|
|
|
|
|
(states (list initial-state))
|
|
|
|
|
(next-state-number 1))
|
|
|
|
|
(setf (gethash initial-kernel lalr-states-hash) initial-state)
|
|
|
|
|
(do ((source-states (list initial-state)))
|
|
|
|
|
((endp source-states))
|
|
|
|
|
(let ((source-state (pop source-states)))
|
|
|
|
|
(state-each-shift-item-alist
|
|
|
|
|
#'(lambda (shift-symbol kernel kernel-item-alist)
|
|
|
|
|
(let ((destination-state (gethash kernel lalr-states-hash)))
|
|
|
|
|
(if destination-state
|
|
|
|
|
(dolist (acons kernel-item-alist)
|
|
|
|
|
(pushnew (state-laitem destination-state (car acons)) (laitem-propagates (cdr acons))))
|
|
|
|
|
(progn
|
|
|
|
|
(setq destination-state (make-state grammar kernel kernel-item-alist next-state-number *empty-terminalset*))
|
|
|
|
|
(setf (gethash kernel lalr-states-hash) destination-state)
|
|
|
|
|
(incf next-state-number)
|
|
|
|
|
(push destination-state states)
|
|
|
|
|
(push destination-state source-states)))
|
|
|
|
|
(if (nonterminal? shift-symbol)
|
|
|
|
|
(push (cons shift-symbol destination-state)
|
|
|
|
|
(state-gotos source-state))
|
|
|
|
|
(push (cons shift-symbol (make-shift-transition destination-state))
|
|
|
|
|
(state-transitions source-state)))))
|
|
|
|
|
source-state)))
|
|
|
|
|
(setf (grammar-states grammar) (nreverse states))
|
|
|
|
|
initial-state))
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
; Propagate the lookaheads in the LALR(1) grammar.
|
|
|
|
|
(defun propagate-lalr-lookaheads (grammar)
|
|
|
|
|
(let ((dirty-laitems (make-hash-table :test #'eq)))
|
|
|
|
|
(dolist (state (grammar-states grammar))
|
|
|
|
|
(dolist (laitem (state-laitems state))
|
|
|
|
|
(when (and (laitem-propagates laitem) (not (terminalset-empty? (laitem-lookaheads laitem))))
|
|
|
|
|
(setf (gethash laitem dirty-laitems) t))))
|
|
|
|
|
(do ()
|
|
|
|
|
((zerop (hash-table-count dirty-laitems)))
|
|
|
|
|
(dolist (dirty-laitem (hash-table-keys dirty-laitems))
|
|
|
|
|
(remhash dirty-laitem dirty-laitems)
|
|
|
|
|
(let ((src-lookaheads (laitem-lookaheads dirty-laitem)))
|
|
|
|
|
(dolist (dst-laitem (laitem-propagates dirty-laitem))
|
|
|
|
|
(let* ((old-dst-lookaheads (laitem-lookaheads dst-laitem))
|
|
|
|
|
(new-dst-lookaheads (terminalset-union old-dst-lookaheads src-lookaheads)))
|
|
|
|
|
(unless (terminalset-= old-dst-lookaheads new-dst-lookaheads)
|
|
|
|
|
(setf (laitem-lookaheads dst-laitem) new-dst-lookaheads)
|
|
|
|
|
(setf (gethash dst-laitem dirty-laitems) t)))))))
|
|
|
|
|
|
|
|
|
|
;Erase the propagates chains in all laitems.
|
|
|
|
|
(dolist (state (grammar-states grammar))
|
|
|
|
|
(dolist (laitem (state-laitems state))
|
|
|
|
|
(setf (laitem-propagates laitem) nil)))))
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
;;; ------------------------------------------------------------------------------------------------------
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
; Calculate the reduce and accept transitions in the grammar.
|
|
|
|
|
; Also sort all transitions by their terminal numbers and gotos by their nonterminal numbers.
|
|
|
|
|
; Conflicting transitions are sorted as follows:
|
|
|
|
|
; shifts come before reduces and accepts
|
|
|
|
|
; accepts come before reduces
|
|
|
|
|
; reduces with lower production numbers come before reduces with higher production numbers
|
|
|
|
|
; Disambiguation will choose the first member of a sorted list of conflicting transitions.
|
|
|
|
|
(defun finish-transitions (grammar)
|
|
|
|
|
(dolist (state (grammar-states grammar))
|
|
|
|
|
(dolist (laitem (state-laitems state))
|
|
|
|
|
(let ((item (laitem-item laitem)))
|
|
|
|
|
(unless (item-next-symbol item)
|
|
|
|
|
(if (grammar-symbol-= (item-lhs item) *start-nonterminal*)
|
|
|
|
|
(when (terminal-in-terminalset grammar *end-marker* (laitem-lookaheads laitem))
|
|
|
|
|
(push (cons *end-marker* (make-accept-transition))
|
|
|
|
|
(state-transitions state)))
|
|
|
|
|
(map-terminalset-reverse
|
|
|
|
|
#'(lambda (lookahead)
|
|
|
|
|
(push (cons lookahead (make-reduce-transition (item-production item)))
|
|
|
|
|
(state-transitions state)))
|
|
|
|
|
grammar
|
|
|
|
|
(laitem-lookaheads laitem))))))
|
|
|
|
|
(setf (state-gotos state)
|
|
|
|
|
(sort (state-gotos state) #'< :key #'(lambda (goto-cons) (state-number (cdr goto-cons)))))
|
|
|
|
|
(setf (state-transitions state)
|
|
|
|
|
(sort (state-transitions state)
|
|
|
|
|
#'(lambda (transition-cons-1 transition-cons-2)
|
|
|
|
|
(let ((terminal-number-1 (terminal-number grammar (car transition-cons-1)))
|
|
|
|
|
(terminal-number-2 (terminal-number grammar (car transition-cons-2))))
|
|
|
|
|
(cond
|
|
|
|
|
((< terminal-number-1 terminal-number-2) t)
|
|
|
|
|
((> terminal-number-1 terminal-number-2) nil)
|
|
|
|
|
(t (let* ((transition1 (cdr transition-cons-1))
|
|
|
|
|
(transition2 (cdr transition-cons-2))
|
|
|
|
|
(transition-kind-1 (transition-kind transition1))
|
|
|
|
|
(transition-kind-2 (transition-kind transition2)))
|
|
|
|
|
(cond
|
|
|
|
|
((eq transition-kind-2 :shift) nil)
|
|
|
|
|
((eq transition-kind-1 :shift) t)
|
|
|
|
|
((eq transition-kind-2 :accept) nil)
|
|
|
|
|
((eq transition-kind-1 :accept) t)
|
|
|
|
|
(t (let ((production-number-1 (production-number (transition-production transition1)))
|
|
|
|
|
(production-number-2 (production-number (transition-production transition2))))
|
|
|
|
|
(< production-number-1 production-number-2)))))))))))))
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
; Find ambiguities, if any, in the grammar. Report them on the given stream.
|
|
|
|
|
; Fix all ambiguities in favor of the first transition listed
|
|
|
|
|
; (the transitions were ordered by finish-transitions).
|
|
|
|
|
(defun report-and-fix-ambiguities (grammar stream)
|
|
|
|
|
(let ((found-ambiguities nil))
|
|
|
|
|
(pprint-logical-block (stream nil)
|
|
|
|
|
(dolist (state (grammar-states grammar))
|
|
|
|
|
(labels
|
|
|
|
|
|
|
|
|
|
((report-ambiguity (transition-cons other-transition-conses)
|
|
|
|
|
(unless found-ambiguities
|
|
|
|
|
(setq found-ambiguities t)
|
|
|
|
|
(format stream "~&Ambiguities:")
|
|
|
|
|
(pprint-indent :block 2 stream))
|
|
|
|
|
(pprint-newline :mandatory stream)
|
|
|
|
|
(pprint-logical-block (stream nil)
|
|
|
|
|
(format stream "S~D: ~W ~:_=> ~:_" (state-number state) (car transition-cons))
|
|
|
|
|
(pprint-logical-block (stream nil)
|
|
|
|
|
(dolist (a (cons transition-cons other-transition-conses))
|
|
|
|
|
(print-transition (cdr a) stream)
|
|
|
|
|
(format stream " ~:_")))))
|
|
|
|
|
|
|
|
|
|
; Check the list of transition-conses and report ambiguities.
|
|
|
|
|
; start is the start of a possibly larger list of transition-conses whose tail
|
|
|
|
|
; is the given list. If ambiguities exist, return a copy of start up to the
|
|
|
|
|
; position of list in it followed by list with ambiguities removed. If not,
|
|
|
|
|
; return start unchanged.
|
|
|
|
|
(check (transition-conses start)
|
|
|
|
|
(if transition-conses
|
|
|
|
|
(let* ((transition-cons (first transition-conses))
|
|
|
|
|
(transition-terminal (car transition-cons))
|
|
|
|
|
(transition-conses-rest (rest transition-conses)))
|
|
|
|
|
(if transition-conses-rest
|
|
|
|
|
(if (grammar-symbol-= transition-terminal (car (first transition-conses-rest)))
|
|
|
|
|
(let ((unrelated-transitions
|
|
|
|
|
(member-if #'(lambda (a) (not (grammar-symbol-= transition-terminal (car a))))
|
|
|
|
|
transition-conses-rest)))
|
|
|
|
|
(report-ambiguity transition-cons (ldiff transition-conses-rest unrelated-transitions))
|
|
|
|
|
(check unrelated-transitions (append (ldiff start transition-conses-rest) unrelated-transitions)))
|
|
|
|
|
(check transition-conses-rest start))
|
|
|
|
|
start))
|
|
|
|
|
start)))
|
|
|
|
|
|
|
|
|
|
(let ((transition-conses (state-transitions state)))
|
|
|
|
|
(setf (state-transitions state) (check transition-conses transition-conses))))))
|
|
|
|
|
(when found-ambiguities
|
|
|
|
|
(pprint-newline :mandatory stream))))
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
; Erase the existing parser, if any, for the given grammar.
|
|
|
|
|
(defun clear-parser (grammar)
|
|
|
|
|
(clrhash (grammar-items-hash grammar))
|
|
|
|
|
(setf (grammar-states grammar) nil))
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
; Construct a LR or LALR parser in the given grammar. kind should be either :lalr-1 or :lr-1.
|
|
|
|
|
; Return the grammar.
|
|
|
|
|
(defun compile-parser (grammar kind)
|
|
|
|
|
(clear-parser grammar)
|
|
|
|
|
(ecase kind
|
|
|
|
|
(:lalr-1
|
|
|
|
|
(add-all-lalr-states grammar)
|
|
|
|
|
(propagate-lalr-lookaheads grammar))
|
|
|
|
|
(:lr-1
|
|
|
|
|
(add-all-lr-states grammar)))
|
|
|
|
|
(finish-transitions grammar)
|
|
|
|
|
(report-and-fix-ambiguities grammar *error-output*)
|
|
|
|
|
grammar)
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
; Make the grammar and compile its parser. kind should be either :lalr-1 or :lr-1.
|
1999-02-10 06:39:58 +00:00
|
|
|
(defun make-and-compile-grammar (kind parametrization start-symbol grammar-source &optional excluded-nonterminals-source)
|
|
|
|
|
(compile-parser (make-grammar parametrization start-symbol grammar-source excluded-nonterminals-source)
|
1999-01-30 01:19:00 +00:00
|
|
|
kind))
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
;;; ------------------------------------------------------------------------------------------------------
|
|
|
|
|
|
|
|
|
|
; Parse the input list of tokens to produce a parse tree.
|
|
|
|
|
; token-terminal is a function that returns a terminal symbol when given an input token.
|
|
|
|
|
(defun parse (grammar token-terminal input)
|
|
|
|
|
(labels
|
|
|
|
|
(;Continue the parse with the given parser stack and remainder of input.
|
|
|
|
|
(parse-step (stack input)
|
|
|
|
|
(if (endp input)
|
|
|
|
|
(parse-step-1 stack *end-marker* nil nil)
|
|
|
|
|
(let ((token (first input)))
|
|
|
|
|
(parse-step-1 stack (funcall token-terminal token) token (rest input)))))
|
|
|
|
|
|
|
|
|
|
;Same as parse-step except that the next input terminal has been determined already.
|
|
|
|
|
;input-rest contains the input tokens after the next token.
|
|
|
|
|
(parse-step-1 (stack terminal token input-rest)
|
|
|
|
|
(let* ((state (caar stack))
|
|
|
|
|
(transition (cdr (assoc terminal (state-transitions state) :test *grammar-symbol-=*))))
|
|
|
|
|
(if transition
|
|
|
|
|
(case (transition-kind transition)
|
|
|
|
|
(:shift (parse-step (acons (transition-state transition) token stack) input-rest))
|
|
|
|
|
(:reduce (let ((production (transition-production transition))
|
|
|
|
|
(expansion nil))
|
|
|
|
|
(dotimes (i (production-rhs-length production))
|
|
|
|
|
(push (cdr (pop stack)) expansion))
|
|
|
|
|
(let* ((state (caar stack))
|
|
|
|
|
(dst-state (assert-non-null
|
|
|
|
|
(cdr (assoc (production-lhs production) (state-gotos state) :test *grammar-symbol-=*))))
|
|
|
|
|
(named-expansion (cons (production-name production) expansion)))
|
|
|
|
|
(parse-step-1 (acons dst-state named-expansion stack) terminal token input-rest))))
|
|
|
|
|
(:accept (cdar stack))
|
|
|
|
|
(t (error "Bad transition: ~S" transition)))
|
|
|
|
|
(error "Parse error on ~S followed by ~S ..." token (ldiff input-rest (nthcdr 10 input-rest)))))))
|
|
|
|
|
|
|
|
|
|
(parse-step (list (cons (grammar-start-state grammar) nil)) input)))
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
;;; ------------------------------------------------------------------------------------------------------
|
|
|
|
|
;;; ACTIONS
|
|
|
|
|
|
|
|
|
|
; Initialize the action-signatures hash table, setting each grammar symbol's signature
|
|
|
|
|
; to null for now. Also clear all production actions in the grammar.
|
|
|
|
|
(defun clear-actions (grammar)
|
|
|
|
|
(let ((action-signatures (make-hash-table :test *grammar-symbol-=*))
|
|
|
|
|
(terminals (grammar-terminals grammar))
|
|
|
|
|
(nonterminals (grammar-nonterminals grammar)))
|
|
|
|
|
(dotimes (i (length terminals))
|
|
|
|
|
(setf (gethash (svref terminals i) action-signatures) nil))
|
|
|
|
|
(dotimes (i (length nonterminals))
|
|
|
|
|
(setf (gethash (svref nonterminals i) action-signatures) nil))
|
|
|
|
|
(setf (grammar-action-signatures grammar) action-signatures)
|
|
|
|
|
(each-grammar-production
|
|
|
|
|
grammar
|
|
|
|
|
#'(lambda (production)
|
|
|
|
|
(setf (production-actions production) nil)
|
|
|
|
|
(setf (production-n-action-args production) nil)
|
|
|
|
|
(setf (production-evaluator-code production) nil)
|
|
|
|
|
(setf (production-evaluator production) nil)))
|
|
|
|
|
(clrhash (grammar-terminal-actions grammar))))
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
; Declare the type of action action-symbol, when called on general-grammar-symbol, to be type-expr.
|
|
|
|
|
; Signal an error on duplicate actions.
|
|
|
|
|
; It's OK if some of the symbol instances don't exist, as long as at least one does.
|
|
|
|
|
(defun declare-action (grammar general-grammar-symbol action-symbol type-expr)
|
|
|
|
|
(unless (and action-symbol (symbolp action-symbol))
|
|
|
|
|
(error "Bad action name ~S" action-symbol))
|
|
|
|
|
(let ((action-signatures (grammar-action-signatures grammar))
|
|
|
|
|
(grammar-symbols (general-grammar-symbol-instances grammar general-grammar-symbol))
|
|
|
|
|
(symbol-exists nil))
|
|
|
|
|
(dolist (grammar-symbol grammar-symbols)
|
|
|
|
|
(let ((signature (gethash grammar-symbol action-signatures :undefined)))
|
|
|
|
|
(unless (eq signature :undefined)
|
|
|
|
|
(setq symbol-exists t)
|
|
|
|
|
(when (assoc action-symbol signature :test #'eq)
|
|
|
|
|
(error "Attempt to redefine the type of action ~S on ~S" action-symbol grammar-symbol))
|
|
|
|
|
(setf (gethash grammar-symbol action-signatures)
|
|
|
|
|
(nconc signature (list (cons action-symbol type-expr))))
|
|
|
|
|
(if (nonterminal? grammar-symbol)
|
|
|
|
|
(dolist (production (rule-productions (grammar-rule grammar grammar-symbol)))
|
|
|
|
|
(setf (production-actions production)
|
|
|
|
|
(nconc (production-actions production) (list (cons action-symbol nil)))))
|
|
|
|
|
(let ((terminal-actions (grammar-terminal-actions grammar)))
|
|
|
|
|
(assert-type grammar-symbol terminal)
|
|
|
|
|
(setf (gethash grammar-symbol terminal-actions)
|
|
|
|
|
(nconc (gethash grammar-symbol terminal-actions) (list (cons action-symbol nil)))))))))
|
|
|
|
|
(unless symbol-exists
|
|
|
|
|
(error "Bad action grammar symbol ~S" grammar-symbols))))
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
; Return the list of pairs (action-symbol . type-or-type-expr) for this grammar-symbol.
|
|
|
|
|
; The pairs are in order from oldest to newest action-symbols added to this grammar-symbol.
|
|
|
|
|
(declaim (inline grammar-symbol-signature))
|
|
|
|
|
(defun grammar-symbol-signature (grammar grammar-symbol)
|
|
|
|
|
(gethash grammar-symbol (grammar-action-signatures grammar)))
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
; Return the list of action types of the grammar's user start-symbol.
|
|
|
|
|
(defun grammar-user-start-action-types (grammar)
|
|
|
|
|
(mapcar #'cdr (grammar-symbol-signature grammar (gramar-user-start-symbol grammar))))
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
; If action action-symbol is declared on grammar-symbol, return two values:
|
|
|
|
|
; t, and
|
|
|
|
|
; the action's type-expr;
|
|
|
|
|
; If not, return nil.
|
|
|
|
|
(defun action-declaration (grammar grammar-symbol action-symbol)
|
|
|
|
|
(let ((declaration (assoc action-symbol (grammar-symbol-signature grammar grammar-symbol) :test #'eq)))
|
|
|
|
|
(and declaration
|
|
|
|
|
(values t (cdr declaration)))))
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
; Call f on every action declaration, passing it two arguments:
|
|
|
|
|
; the grammar-symbol;
|
|
|
|
|
; a pair (action-symbol . type-expr).
|
|
|
|
|
; f may modify the action's type-expr.
|
|
|
|
|
(defun each-action-declaration (grammar f)
|
|
|
|
|
(maphash #'(lambda (grammar-symbol signature)
|
|
|
|
|
(dolist (action-declaration signature)
|
|
|
|
|
(funcall f grammar-symbol action-declaration)))
|
|
|
|
|
(grammar-action-signatures grammar)))
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
; Define action action-symbol, when called on the production with the given name,
|
|
|
|
|
; to be action-expr. The action should have been declared already.
|
|
|
|
|
(defun define-action (grammar production-name action-symbol action-expr)
|
|
|
|
|
(dolist (production (general-production-productions (grammar-general-production grammar production-name)))
|
|
|
|
|
(let ((definition (assoc action-symbol (production-actions production) :test #'eq)))
|
|
|
|
|
(cond
|
|
|
|
|
((null definition)
|
|
|
|
|
(error "Attempt to define action ~S on ~S, which hasn't been declared yet" action-symbol production-name))
|
|
|
|
|
((cdr definition)
|
|
|
|
|
(error "Duplicate definition of action ~S on ~S" action-symbol production-name))
|
|
|
|
|
(t (setf (cdr definition) (make-action action-expr)))))))
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
; Define action action-symbol, when called on the given terminal,
|
|
|
|
|
; to execute the given function, which should take a token as an input and
|
|
|
|
|
; produce a value of the proper type as output.
|
|
|
|
|
; The action should have been declared already.
|
|
|
|
|
(defun define-terminal-action (grammar terminal action-symbol action-function)
|
|
|
|
|
(assert-type action-function function)
|
|
|
|
|
(let ((definition (assoc action-symbol (gethash terminal (grammar-terminal-actions grammar)) :test #'eq)))
|
|
|
|
|
(cond
|
|
|
|
|
((null definition)
|
|
|
|
|
(error "Attempt to define action ~S on ~S, which hasn't been declared yet" action-symbol terminal))
|
|
|
|
|
((cdr definition)
|
|
|
|
|
(error "Duplicate definition of action ~S on ~S" action-symbol terminal))
|
|
|
|
|
(t (setf (cdr definition) action-function)))))
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
; Parse the input list of tokens to produce a list of action results.
|
|
|
|
|
; token-terminal is a function that returns a terminal symbol when given an input token.
|
|
|
|
|
; If trace is:
|
|
|
|
|
; nil, don't print trace information
|
|
|
|
|
; :code, print trace information, including action code
|
|
|
|
|
; other print trace information
|
|
|
|
|
; Return two values:
|
|
|
|
|
; the list of action results;
|
|
|
|
|
; the list of action results' types.
|
|
|
|
|
(defun action-parse (grammar token-terminal input &key trace)
|
|
|
|
|
(labels
|
|
|
|
|
(;Continue the parse with the given stacks and remainder of input.
|
|
|
|
|
(parse-step (state-stack value-stack input)
|
|
|
|
|
(if (endp input)
|
|
|
|
|
(parse-step-1 state-stack value-stack *end-marker* nil nil)
|
|
|
|
|
(let ((token (first input)))
|
|
|
|
|
(parse-step-1 state-stack value-stack (funcall token-terminal token) token (rest input)))))
|
|
|
|
|
|
|
|
|
|
;Same as parse-step except that the next input terminal has been determined already.
|
|
|
|
|
;input-rest contains the input tokens after the next token.
|
|
|
|
|
(parse-step-1 (state-stack value-stack terminal token input-rest)
|
|
|
|
|
(let* ((state (car state-stack))
|
|
|
|
|
(transition (cdr (assoc terminal (state-transitions state) :test *grammar-symbol-=*))))
|
|
|
|
|
(if transition
|
|
|
|
|
(case (transition-kind transition)
|
|
|
|
|
(:shift
|
|
|
|
|
(dolist (action-function-binding (gethash terminal (grammar-terminal-actions grammar)))
|
|
|
|
|
(push (funcall (cdr action-function-binding) token) value-stack))
|
|
|
|
|
(parse-step (cons (transition-state transition) state-stack) value-stack input-rest))
|
|
|
|
|
(:reduce
|
|
|
|
|
(let* ((production (transition-production transition))
|
|
|
|
|
(state-stack (nthcdr (production-rhs-length production) state-stack))
|
|
|
|
|
(state (car state-stack))
|
|
|
|
|
(dst-state (assert-non-null
|
|
|
|
|
(cdr (assoc (production-lhs production) (state-gotos state) :test *grammar-symbol-=*))))
|
|
|
|
|
(value-stack (funcall (production-evaluator production) value-stack)))
|
|
|
|
|
(parse-step-1 (cons dst-state state-stack) value-stack terminal token input-rest)))
|
|
|
|
|
(:accept (values (nreverse value-stack) (grammar-user-start-action-types grammar)))
|
|
|
|
|
(t (error "Bad transition: ~S" transition)))
|
|
|
|
|
(error "Parse error on ~S followed by ~S ..." token (ldiff input-rest (nthcdr 10 input-rest)))))))
|
|
|
|
|
|
|
|
|
|
(if trace
|
|
|
|
|
(trace-action-parse grammar token-terminal input trace)
|
|
|
|
|
(parse-step (list (grammar-start-state grammar)) nil input))))
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
; Same as action-parse, but with tracing information
|
|
|
|
|
; If trace is:
|
|
|
|
|
; :code, print trace information, including action code
|
|
|
|
|
; other print trace information
|
|
|
|
|
; Return two values:
|
|
|
|
|
; the list of action results;
|
|
|
|
|
; the list of action results' types.
|
|
|
|
|
(defun trace-action-parse (grammar token-terminal input trace)
|
|
|
|
|
(labels
|
|
|
|
|
(;Continue the parse with the given stacks and remainder of input.
|
|
|
|
|
;type-stack contains the types of corresponding value-stack entries.
|
|
|
|
|
(parse-step (state-stack value-stack type-stack input)
|
|
|
|
|
(if (endp input)
|
|
|
|
|
(parse-step-1 state-stack value-stack type-stack *end-marker* nil nil)
|
|
|
|
|
(let ((token (first input)))
|
|
|
|
|
(parse-step-1 state-stack value-stack type-stack (funcall token-terminal token) token (rest input)))))
|
|
|
|
|
|
|
|
|
|
;Same as parse-step except that the next input terminal has been determined already.
|
|
|
|
|
;input-rest contains the input tokens after the next token.
|
|
|
|
|
(parse-step-1 (state-stack value-stack type-stack terminal token input-rest)
|
|
|
|
|
(let* ((state (car state-stack))
|
|
|
|
|
(transition (cdr (assoc terminal (state-transitions state) :test *grammar-symbol-=*))))
|
|
|
|
|
(format *trace-output* "S~D: ~@_" (state-number state))
|
|
|
|
|
(print-values (reverse value-stack) (reverse type-stack) *trace-output*)
|
|
|
|
|
(pprint-newline :mandatory *trace-output*)
|
|
|
|
|
(if transition
|
|
|
|
|
(case (transition-kind transition)
|
|
|
|
|
(:shift
|
|
|
|
|
(format *trace-output* " shift ~W~:@_" terminal)
|
|
|
|
|
(dolist (action-function-binding (gethash terminal (grammar-terminal-actions grammar)))
|
|
|
|
|
(push (funcall (cdr action-function-binding) token) value-stack))
|
|
|
|
|
(dolist (action-signature (grammar-symbol-signature grammar terminal))
|
|
|
|
|
(push (cdr action-signature) type-stack))
|
|
|
|
|
(parse-step (cons (transition-state transition) state-stack) value-stack type-stack input-rest))
|
|
|
|
|
(:reduce
|
|
|
|
|
(let ((production (transition-production transition)))
|
|
|
|
|
(write-string " reduce " *trace-output*)
|
|
|
|
|
(if (eq trace :code)
|
|
|
|
|
(write production :stream *trace-output* :pretty t)
|
|
|
|
|
(print-production production *trace-output*))
|
|
|
|
|
(pprint-newline :mandatory *trace-output*)
|
|
|
|
|
(let* ((state-stack (nthcdr (production-rhs-length production) state-stack))
|
|
|
|
|
(state (car state-stack))
|
|
|
|
|
(dst-state (assert-non-null
|
|
|
|
|
(cdr (assoc (production-lhs production) (state-gotos state) :test *grammar-symbol-=*))))
|
|
|
|
|
(value-stack (funcall (production-evaluator production) value-stack))
|
|
|
|
|
(type-stack (nthcdr (production-n-action-args production) type-stack)))
|
|
|
|
|
(dolist (action-signature (grammar-symbol-signature grammar (production-lhs production)))
|
|
|
|
|
(push (cdr action-signature) type-stack))
|
|
|
|
|
(parse-step-1 (cons dst-state state-stack) value-stack type-stack terminal token input-rest))))
|
|
|
|
|
(:accept
|
|
|
|
|
(format *trace-output* " accept~:@_")
|
|
|
|
|
(values (nreverse value-stack) (nreverse type-stack)))
|
|
|
|
|
(t (error "Bad transition: ~S" transition)))
|
|
|
|
|
(error "Parse error on ~S followed by ~S ..." token (ldiff input-rest (nthcdr 10 input-rest)))))))
|
|
|
|
|
|
|
|
|
|
(parse-step (list (grammar-start-state grammar)) nil nil input)))
|
|
|
|
|
|
