Change string handling, some debug stuff

src/main.lisp

@@ -1,157 +1,157 b''
 
 (in-package txt2web)
 
 (defvar *app-name* "txt2web")
 
 (defun entry-point-no-args ()
   (setf *delivered* t)
   (entry-point uiop:*command-line-arguments*))
 
 (defun entry-point (args)
   (let ((*package* (find-package :txt2web)))
     (catch :terminate
       (let ((compiler (apply #'make-instance 'compiler (parse-opts args))))
         (write-compiled-file compiler))))
   (values))
 
 (defun parse-opts (args)
   (let ((mode :sources)
         (data (list :sources nil :target nil :js nil :css nil :body nil :compile nil :beautify nil)))
     (loop :for arg :in args
        :do (alexandria:switch (arg :test #'string=)
              ("-o" (setf mode :target))
              ("--js" (setf mode :js))
              ("--css" (setf mode :css))
              ("--body" (setf mode :body))
              ("-c" (setf (getf data :compile) t))
              ("--beautify" (setf (getf data :beautify) t))
              (t (push arg (getf data mode)))))
     (unless (< 0 (length (getf data :sources)))
       (report-error "There should be at least one source"))
     (unless (> 1 (length (getf data :target)))
       (report-error "There should be no more than one target"))
     (unless (> 1 (length (getf data :body)))
       (report-error "There should be no more than one body"))
     (unless (getf data :target)
       (setf (getf data :target)
             (let* ((sources (first (getf data :sources)))
                    (tokens (uiop:split-string sources :separator "."))
                    (target (format nil "~{~A~^.~}.html"
                                    (butlast tokens))))
               (list target))))
     (list :sources (getf data :sources)
           :target (first (getf data :target))
           :js (getf data :js)
           :css (getf data :css)
           :body (first (getf data :body))
           :compile (getf data :compile)
           :beautify (getf data :beautify))))
 
 (defun print-usage ()
   (lformat t :usage *app-name*))
 
 (defun parse-file (filename)
   (handler-case
       (p:parse 'txt2web-grammar
-               (alexandria:read-file-into-string filename))
+               (alexandria:read-file-into-string filename :external-format :utf-8))
     (p:esrap-parse-error (e)
       (format t "~A~%" e)
       (throw :terminate nil))))
 
 (defun report-error (fmt &rest args)
   (format t "ERROR: ~A~%" (apply #'format nil fmt args))
   (print-usage)
   (throw :terminate nil))
 
 ;;; JS
 
 (defun minify-package (package-designator minify prefix)
   (setf (ps:ps-package-prefix package-designator) prefix)
   (if minify
       (ps:obfuscate-package package-designator)
       (ps:unobfuscate-package package-designator)))
 
 (defmethod js-sources ((compiler compiler))
   (let ((ps:*ps-print-pretty* (beautify compiler)))
     (cond ((beautify compiler)
            (minify-package "TXT2WEB.MAIN" nil "qsp_")
            (minify-package "TXT2WEB.API"  nil "qsp_api_")
            (minify-package "TXT2WEB.LIB"  nil "qsp_lib_"))
           (t
            (minify-package "TXT2WEB.MAIN" t   "_")
            (minify-package "TXT2WEB.API"  t   "a_")
            (minify-package "TXT2WEB.LIB"  t   "l_")))
     (format nil "~{~A~^~%~%~}" (mapcar #'ps:ps* (reverse (js compiler))))))
 
 ;;; CSS
 
 (defmethod css-sources ((compiler compiler))
   (format nil "~{~A~^~%~%~}" (css compiler)))
 
 ;;; HTML
 
 (defmethod html-sources ((compiler compiler))
   (let ((flute:*escape-html* nil)
         (body-template (body compiler))
         (js (js-sources compiler))
         (css (css-sources compiler)))
     (with-output-to-string (out)
       (write
        (flute:h
          (html
           (head
            (title "txt2web"))
           (body
            body-template
            (style css)
            (script js))))
        :stream out
        :pretty nil))))
 
 (defun filename-game (filename)
   (let ((filename (alexandria:lastcar (uiop:split-string filename :separator "/"))))
     (format nil "~{~A~^.~}" (butlast (uiop:split-string filename :separator ".")))))
 
 (defmethod initialize-instance ((compiler compiler) &key sources ((:js js-files)) ((:css css-files)) ((:body body-file)) compile &allow-other-keys)
   (call-next-method)
   (with-slots (body css js)
       compiler
     ;; Compile the game's JS
     (dolist (source sources)
       (let ((ps (parse-file source))
             (game-name (filename-game source)))
         (destructuring-bind (kw &rest locations)
             ps
           (unless (eq kw 'lib:game)
             (report-error "Internal error!"))
           (push 
            `(lib:game (,game-name) ,@locations)
            js))))
     ;; Does the user need us to do anything else
     (unless compile
       ;; Read in body
       (when body-file
         (setf body
-              (alexandria:read-file-into-string body-file)))
+              (alexandria:read-file-into-string body-file :external-format :utf-8)))
       ;; Include js files
       (dolist (js-file js-files)
         (push (format nil "////// Included file ~A~%~A" js-file
-                      (alexandria:read-file-into-string js-file))
+                      (alexandria:read-file-into-string js-file :external-format :utf-8))
               js))
       ;; Include css files
       (when css-files
         ;; User option overrides the default css
         (setf css nil)
         (dolist (css-file css-files)
           (push (format nil "////// Included file ~A~%~A" css-file
-                        (alexandria:read-file-into-string css-file))
+                        (alexandria:read-file-into-string css-file :external-format :utf-8))
                 css))))))
 
 (defmethod write-compiled-file ((compiler compiler))
   (alexandria:write-string-into-file
    (if (compile-only compiler)
        ;; Just the JS
        (js-sources compiler)
        ;; All of it
        (html-sources compiler))
    (target compiler) :if-exists :supersede))

src/parser.lisp

@@ -1,629 +1,660 b''
 
 (in-package txt2web)
 
 ;;;; Parses TXT source to an intermediate representation
 
 (eval-when (:compile-toplevel :load-toplevel :execute)
   (defparameter *max-args* 10))
 
 ;;; Utility
 
 (defun remove-nth (list nth)
   (append (subseq list 0 nth)
           (subseq list (1+ nth))))
 
 (defun not-quote (char)
   (not (eql #\' char)))
 
 (defun not-doublequote (char)
   (not (eql #\" char)))
 
 (defun not-brace (char)
   (not (eql #\} char)))
 
 (defun not-integer (string)
   (when (find-if-not #'digit-char-p string)
     t))
 
 (defun not-newline (char)
   (not (eql #\newline char)))
 
 (defun id-any-char (char)
   (and
    (not (digit-char-p char))
    (not (eql #\newline char))
    (not (find char " 	!:&=<>+-*/,'\"()[]{}"))))
 
 (defun intern-first (list)
   (list* (intern (string-upcase (first list)) "TXT2WEB.LIB")
          (rest list)))
 
 (eval-when (:compile-toplevel :load-toplevel :execute)
   (defun remove-nil (list)
     (remove nil list)))
 
 (defun binop-rest (list)
   (destructuring-bind (ws1 operator ws2 operand2)
       list
     (declare (ignore ws1 ws2))
     (list (intern (string-upcase operator) "TXT2WEB.LIB") operand2)))
 
 (defun do-binop% (left-op other-ops)
   (if (null other-ops)
       left-op
       (destructuring-bind ((operator right-op) &rest rest-ops)
           other-ops
         (if (and (listp left-op)
                  (eq (first left-op)
                      operator))
             (do-binop% (append left-op (list right-op)) rest-ops)
             (do-binop% (list operator left-op right-op) rest-ops)))))
 
 (walker:deftransform parser-qspmod mod (&rest args)
   (list* 'qspmod (mapcar #'walker:walk-continue args)))
 
 (defun do-binop (list)
   (walker:walk 'parser-qspmod
                (destructuring-bind (left-op rest-ops)
                    list
                  (do-binop% left-op
                    (mapcar #'binop-rest rest-ops)))))
 
+(defun maybe-text (list)
+  "Leaves lists in place and applies esrap:text to everything else"
+  (let ((parts nil)
+        (part (list 'text)))
+    (loop :for token :in list
+       :do (cond ((listp token)
+                  (push (nreverse part) parts)
+                  (setf part (list 'text))
+                  (push token parts))
+                 (t (push token part))))
+    (push (nreverse part) parts)
+    (remove ""
+            (loop :for part :in (nreverse parts)
+               :collect (case (first part)
+                          ('text (p:text (rest part)))
+                          (t part)))
+            :test #'equal)))
+
 (p:defrule line-continuation (and #\_ #\newline)
   (:constant nil))
 
 (p:defrule text-spaces (+ (or #\space #\tab line-continuation))
   (:text t))
 
 (p:defrule spaces (+ (or #\space #\tab line-continuation))
   (:constant nil)
   (:error-report nil))
 
 (p:defrule spaces? (* (or #\space #\tab line-continuation))
   (:constant nil)
   (:error-report nil))
 
 (p:defrule colon #\:
   (:constant nil))
 
 (p:defrule equal #\=
   (:constant nil))
 
 (p:defrule alphanumeric (alphanumericp character))
 
 (p:defrule not-newline (not-newline character))
 
 (p:defrule squote-esc "''"
   (:lambda (list)
     (p:text (elt list 0))))
 
 (p:defrule dquote-esc "\"\""
   (:lambda (list)
     (p:text (elt list 0))))
 
-(p:defrule sstring-chars (+ (and (and (p:! "<<") (p:! (p:~ "\"exec:")))
-                                 (or squote-esc (not-quote character))))
-  (:lambda (list)
-    (p:text (mapcar #'second list))))
+(p:defrule sstring-char (or squote-esc (not-quote character))
+  (:text t))
 
-(p:defrule dstring-chars (+ (and (and (p:! "<<") (p:! (p:~ "'exec:")))
-                                 (or dquote-esc (not-doublequote character))))
-  (:lambda (list)
-    (p:text (mapcar #'second list))))
+(p:defrule dstring-char (or dquote-esc (not-doublequote character))
+  (:text t))
 
 ;;; Identifiers
 
 (defparameter *keywords* '(act addlib addobj addqst and arrcomp arrpos arrsize cla clear *clear close clr *clr cls cmdclear cmdclr copyarr countobj curact curacts curloc debug delact dellib delobj desc disablescroll disablesubex dynamic dyneval else elseif end exit for freelib func getobj gosub goto gs gt if iif img *img inclib input instr isnum isplay jump killall killobj killqst killvar lcase len let loc local maintxt max menu mid min mod msecscount msg nl *nl no nosave obj opengame openqst or p *p pl *pl play qspver rand refint replace rgb rnd savegame selact selobj set settimer showacts showinput showobjs showstat stattxt step str strcomp strfind strpos to trim ucase unsel unselect user_text usrtxt val view wait xgoto xgt))
 
 (defun trim-$ (str)
   (if (char= #\$ (elt str 0))
       (subseq str 1)
       str))
 
 (defun qsp-keyword-p (id)
   (member (intern (trim-$ (string-upcase id))) *keywords*))
 
 (defun not-qsp-keyword-p (id)
   (not (member (intern (trim-$ (string-upcase id))) *keywords*)))
 
 (p:defrule qsp-keyword (qsp-keyword-p identifier-raw))
 
 (p:defrule id-first (id-any-char character))
 (p:defrule id-next (or (id-any-char character)
                        (digit-char-p character)))
 (p:defrule identifier-raw (and id-first (* id-next))
   (:lambda (list)
     (intern (string-upcase (p:text list)) "TXT2WEB.LIB")))
 
 (p:defrule identifier (not-qsp-keyword-p identifier-raw))
 
 ;;; Strings
 
 (p:defrule qsp-string (or normal-string brace-string))
 
+(p:defrule brace-string (and #\{ before-statement block-body #\})
+  (:lambda (list)
+    (list* 'lib:qspblock (third list))))
+
 (p:defrule normal-string (or sstring dstring)
   (:lambda (str)
     (list* 'lib:str (or str (list "")))))
 
-(p:defrule sstring (and #\' (* (or string-interpol
+(p:defrule sstring (and #\' (* (or sstring-interpol
                                    sstring-exec
-                                   sstring-chars))
+                                   sstring-char))
                         #\')
-  (:function second))
+  (:lambda (list)
+    (maybe-text (second list))))
 
-(p:defrule dstring (and #\" (* (or string-interpol
+(p:defrule dstring (and #\" (* (or dstring-interpol
                                    dstring-exec
-                                   dstring-chars))
+                                   dstring-char))
                         #\")
-  (:function second))
+  (:lambda (list)
+    (maybe-text (second list))))
 
-(p:defrule string-interpol (and "<<" expression ">>")
-  (:function second))
+(defun parse-interpol (list)
+  (p:parse 'expression (p:text (mapcar 'second (second list)))))
+
+(defun parse-exec (list)
+  (list* 'lib:exec (p:parse 'exec-body (p:text (second list)))))
 
-(p:defrule sstring-exec-body (+ (or squote-esc (not-doublequote character)))
-  (:text t))
+(p:defrule sstring-interpol (and "<<" (+ (and (p:! ">>")
+                                              sstring-char))
+                                 ">>")
+  (:function parse-interpol))
 
-(p:defrule dstring-exec-body (+ (or dquote-esc (not-quote character)))
-  (:text t))
+(p:defrule dstring-interpol (and "<<" (+ (and (p:! ">>")
+                                              dstring-char))
+                                 ">>")
+  (:function parse-interpol))
 
-(p:defrule sstring-exec (and (p:~ "\"exec:") sstring-exec-body #\")
-  (:lambda (list)
-    (list* 'lib:exec (p:parse 'exec-body (second list)))))
+(p:defrule sstring-exec (or (and (p:~ "\"exec:")
+                                 (+ (and (p:& (not-doublequote character)) sstring-char))
+                                 #\")
+                            (and (p:~ "''exec:")
+                                 (+ (not-quote character))
+                                 "''"))
+  (:function parse-exec))
 
-(p:defrule dstring-exec (and (p:~ "'exec:") dstring-exec-body #\')
-  (:lambda (list)
-    (list* 'lib:exec (p:parse 'exec-body (second list)))))
-
-(p:defrule brace-string (and #\{ before-statement block-body #\})
-  (:lambda (list)
-    (list* 'lib:qspblock (third list))))
+(p:defrule dstring-exec (or (and (p:~ "'exec:")
+                                 (+ (and (p:& (not-quote character)) dstring-char))
+                                 #\')
+                            (and (p:~ "\"\"exec")
+                                 (+ (not-doublequote character))
+                                 "\"\""))
+  (:function parse-exec))
 
 ;;; Location
 
 (p:defrule txt2web-grammar (and (* (or spaces #\newline))
                                   (* location))
   (:lambda (list)
     `(lib:game ,@(second list))))
 
 (p:defrule location (and location-header block-body location-end)
   (:destructure (header body end)
     (declare (ignore end))
     `(lib:location (,header) ,@body)))
 
 (p:defrule location-header (and #\#
                                 (+ not-newline)
                                 (and #\newline spaces? before-statement))
   (:destructure (spaces1 name spaces2)
     (declare (ignore spaces1 spaces2))
     (string-upcase (string-trim " " (p:text name)))))
 
 (p:defrule location-end (and #\- (* not-newline) #\newline before-statement)
   (:constant nil))
 
 ;;; Block body
 
 (p:defrule newline-block-body (and #\newline spaces? block-body)
   (:function third))
 
 (p:defrule block-body (* statement)
   (:function remove-nil))
 
 ;; Just for <a href="exec:...'>
 ;; Explicitly called from that rule's production
 (p:defrule exec-body (and before-statement line-body)
   (:function second))
 
 (p:defrule line-body (and inline-statement (* next-inline-statement))
   (:lambda (list)
     (list* (first list) (second list))))
 
 (p:defrule before-statement (* (or #\newline spaces))
   (:constant nil))
 
 (p:defrule statement-end (or statement-end-real statement-end-block-close))
 
 (p:defrule statement-end-real (and (or #\newline
                                        (and #\& spaces? (p:& statement%)))
                                    before-statement)
   (:constant nil))
 
 (p:defrule statement-end-block-close (or (p:& #\}))
   (:constant nil))
 
 (p:defrule inline-statement (and statement% spaces?)
   (:function first))
 
 (p:defrule next-inline-statement (and #\& spaces? inline-statement)
   (:function third))
 
 (p:defrule not-a-non-statement (and (p:! (p:~ "elseif"))
                                     (p:! (p:~ "else"))
                                     (p:! (p:~ "end"))))
 
 (p:defrule statement (and inline-statement statement-end)
   (:function first))
 
 (p:defrule statement% (and not-a-non-statement
                            (or label comment string-output
                                block non-returning-intrinsic local
                                assignment expression-output))
   (:function second))
 
 (p:defrule expr-stopper (or comment block non-returning-intrinsic))
 
 (p:defrule string-output qsp-string
   (:lambda (string)
     (list 'lib:main-pl string)))
 
 (p:defrule expression-output expression
   (:lambda (list)
     (list 'lib:main-pl list)))
 
 (p:defrule label (and colon identifier)
   (:lambda (list)
     (intern (string (second list)) :keyword)))
 
 (p:defrule comment (and #\! (* (or qsp-string brace-comment text-spaces not-newline)))
   (:constant nil))
 
 (p:defrule brace-comment (and #\{ (* (not-brace character)) #\})
   (:constant nil))
 
 (p:defrule local (and (p:~ "local") spaces variable (p:? (and spaces? #\= spaces? expression)))
   (:lambda (list)
     (list* 'lib:local (third list)
            (when (fourth list)
              (list (fourth (fourth list)))))))
 
 ;;; Blocks
 
 (p:defrule block (or block-act block-if block-for))
 
 (p:defrule block-if (and block-if-head block-if-body)
   (:destructure (head body)
     `(lib:qspcond (,@head ,@(first body))
                   ,@(rest body))))
 
 (p:defrule block-if-head (and (p:~ "if") spaces expression spaces? colon spaces?)
   (:function remove-nil)
   (:function cdr))
 
 (p:defrule block-if-body (or block-if-ml block-if-sl)
   (:destructure (if-body elseifs else &rest ws)
     (declare (ignore ws))
     `(,(remove-nil if-body) ,@elseifs ,@(when else `((else ,@else))))))
 
 (p:defrule block-if-sl (and line-body
                             (p:? block-if-elseif-inline)
                             (p:? block-if-else-inline)
                             spaces?))
 
 (p:defrule block-if-ml (and (and #\newline spaces?)
                             block-body
                             (p:? block-if-elseif)
                             (p:? block-if-else)
                             block-if-end)
   (:lambda (list)
     (cdr list)))
 
 (p:defrule block-if-elseif-inline (and block-if-elseif-head line-body (p:? block-if-elseif-inline))
   (:destructure (head statements elseif)
     `((,@(rest head) ,@(remove-nil statements)) ,@elseif)))
 
 (p:defrule block-if-elseif (and block-if-elseif-head statement-end block-body (p:? block-if-elseif))
   (:destructure (head ws statements elseif)
     (declare (ignore ws))
     `((,@(rest head) ,@(remove-nil statements)) ,@elseif)))
 
 (p:defrule block-if-elseif-head (and (p:~ "elseif") spaces expression spaces? colon spaces?)
   (:function remove-nil)
   (:function intern-first))
 
 (p:defrule block-if-else-inline (and block-if-else-head line-body)
   (:function second))
 
 (p:defrule block-if-else (and block-if-else-head #\newline spaces? block-body)
   (:function fourth))
 
 (p:defrule block-if-else-head (and (p:~ "else") spaces?)
   (:constant nil))
 
 (p:defrule block-if-end (and (p:~ "end") 
                              (p:? (and spaces (p:~ "if"))))
   (:constant nil))
 
 (p:defrule block-act (and block-act-head (or block-ml block-sl))
   (:lambda (list)
     (apply #'append list)))
 
 (p:defrule block-act-head (and (p:~ "act") spaces? qsp-string spaces?
                                (p:? block-act-head-img)
                                colon spaces?)
   (:lambda (list)
     (intern-first (list (first list)
                         (third list)
                         (or (fifth list) '(lib:str ""))))))
 
 (p:defrule block-act-head-img (and #\, spaces? qsp-string spaces?)
   (:lambda (list)
     (or (third list) "")))
 
 (p:defrule block-for (and block-for-head (or block-ml block-sl))
   (:lambda (list)
     (apply #'append list)))
 
 (p:defrule block-for-head (and (p:~ "for") spaces variable spaces? equal spaces? expression 
                                (p:~ "to") spaces expression
                                block-for-head-step
                                colon spaces?)
   (:lambda (list)
     (list 'lib:qspfor
           (elt list 2)
           (elt list 6)
           (elt list 9)
           (elt list 10))))
 
 (p:defrule block-for-head-step (p:? (and (p:~ "step") spaces expression spaces?))
   (:lambda (list)
     (if list
         (third list)
         1)))
 
 (p:defrule block-sl line-body)
 
 (p:defrule block-ml (and newline-block-body block-end)
   (:lambda (list)
     (apply #'list* (butlast list))))
 
 (p:defrule block-end (and (p:~ "end"))
   (:constant nil))
 
 ;;; Calls
 
 (p:defrule first-argument (and expression spaces?)
   (:function first))
 (p:defrule next-argument (and "," spaces? expression)
   (:function third))
 (p:defrule arguments (or parenthesized-arguments plain-arguments no-arguments))
 (p:defrule parenthesized-arguments (and spaces? #\( base-arguments #\))
   (:function third))
 (p:defrule plain-arguments (and spaces? base-arguments)
   (:function second))
 (p:defrule no-arguments (or (and spaces? (p:& #\newline))
                             (and spaces? (p:& #\&))
                             spaces?)
   (:constant nil))
 (p:defrule base-arguments (or (and first-argument (* next-argument)) spaces?)
   (:lambda (list)
     (if (null list)
         nil
         (list* (first list) (second list)))))
 
 ;;; Intrinsics
 
 (defmacro defintrinsics ((rule-name returning-rule-name non-returning-rule-name) &body clauses)
   `(progn
      ,@(loop :for clause :in clauses
           :collect `(defintrinsic ,@clause))
      (p:defrule ,returning-rule-name (or ,@(remove-nil
                                             (mapcar (lambda (clause)
                                                       (when (second clause)
                                                         (alexandria:symbolicate
                                                          'intrinsic- (first clause))))
                                                     clauses))))
      (p:defrule ,non-returning-rule-name (or ,@(remove-nil
                                                 (mapcar (lambda (clause)
                                                           (unless (second clause)
                                                             (alexandria:symbolicate
                                                              'intrinsic- (first clause))))
                                                         clauses))))
      (p:defrule ,rule-name (or ,returning-rule-name ,non-returning-rule-name))))
 
 (defmacro defintrinsic (sym returning &optional (min-arity 0) max-arity &rest names)
   (declare (ignore returning))
   (unless max-arity
     (setf max-arity *max-args*))
   (setf names 
         (if names
             (mapcar #'string-upcase names)
             (list (string sym))))
   `(p:defrule ,(alexandria:symbolicate 'intrinsic- sym)
        (and (p:? #\$) (or ,@(loop :for name :in names :collect `(p:~ ,name)))
             arguments)
      (:destructure (dollar name arguments)
        (declare (ignore dollar))
        (unless (<= ,min-arity (length arguments) ,max-arity)
          (error "Intrinsic ~A expects between ~A and ~A arguments but ~A were provided:~%~S"
                 name ,min-arity ,max-arity (length arguments) arguments))
        (list* ',(intern (string sym) "TXT2WEB.LIB") arguments))))
 
 (defintrinsics (intrinsic returning-intrinsic non-returning-intrinsic)
   ;; Transitions
   (goto%    nil 0 nil "gt" "goto")
   (xgoto%   nil 0 nil "xgt" "xgoto")
   ;; Variables
   (killvar  nil 0 2)
   ;; Expressions
   (obj      t   1 1)
   (loc      t   1 1)
   (no       t   1 1)
   ;; Basic
   (qspver   t   0 0)
   (curloc   t   0 0)
   (rand     t   1 2)
   (rnd      t   0 0)
   (qspmax   t   1 nil "max")
   (qspmin   t   1 nil "min")
   ;; Arrays
   (killall  nil 0 0)
   (copyarr  nil 2 4)
   (arrsize  t   1 1)
   (arrpos   t   2 3)
   (arrcomp  t   2 3)
   ;; Strings
   (len      t   1 1)
   (mid      t   2 3)
   (ucase    t   1 1)
   (lcase    t   1 1)
   (trim     t   1 1)
   (qspreplace t 2 3 "replace")
   (instr    t   2 3)
   (isnum    t   1 1)
   (val      t   1 1)
   (qspstr   t   1 1 "str")
   (strcomp  t   2 2)
   (strfind  t   2 3)
   (strpos   t   2 3)
   ;; IF
   (iif      t   2 3)
   ;; Subs
   (gosub    nil 1 nil "gosub" "gs")
   (func     t   1 nil)
   (exit     nil 0 0)
   ;; Jump
   (jump     nil 1 1)
   ;; Dynamic
   (dynamic  nil 1 nil)
   (dyneval  t   1 nil)
   ;; Sound
   (play     nil 1 2)
   (isplay   t   1 1)
   (close    nil 1 1)
   (closeall nil 0 0 "close all")
   ;; Main window
   (main-pl  nil 1 1 "*pl")
   (main-nl  nil 0 1 "*nl")
   (main-p   nil 1 1 "*p")
   (maintxt  t   0 0)
   (desc     t   1 1)
   (main-clear   nil 0 0 "*clear" "*clr")
   ;; Aux window
   (showstat nil 1 1)
   (stat-pl  nil 1 1 "pl")
   (stat-nl  nil 0 1 "nl")
   (stat-p   nil 1 1 "p")
   (stattxt  t   0 0)
   (stat-clear    nil 0 0 "clear" "clr")
   (cls      nil 0 0)
   ;; Dialog
   (msg      nil 1 1)
   ;; Acts
   (showacts nil 1 1)
   (delact   nil 1 1 "delact" "del act")
   (curacts  t   0 0)
   (selact   t   0 0)
   (cla      nil 0 0)
   ;; Objects
   (showobjs nil 1 1)
   (addobj   nil 1 3 "addobj" "add obj")
   (delobj   nil 1 1 "delobj" "del obj")
   (killobj  nil 0 1)
   (countobj t   0 0)
   (getobj   t   1 1)
   (selobj   t   0 0)
   (unsel    nil 0 0 "unsel" "unselect")
   ;; Menu
   (menu     nil 1 1)
   ;; Images
   (refint   nil 0 0)
   (view     nil 0 1)
   (img      nil 1)
   (*img     nil 1)
   ;; Fonts
   (rgb      t   3 3)
   ;; Input
   (showinput nil 1 1)
   (usertxt  t   0 0 "user_text" "usrtxt")
   (cmdclear nil 0 0 "cmdclear" "cmdclr")
   (input    t   1 1)
   ;; Files
   (openqst  nil 1 1)
   (addqst   nil 1 1 "addqst" "addlib" "inclib")
   (killqst  nil 1 1 "killqst" "dellib" "freelib")
   (opengame nil 0 0)
   (savegame nil 0 0)
   ;; Real time
   (wait     nil 1 1)
   (msecscount t 0 0)
   (settimer nil 1 1))
 
 ;;; Expression
 
 (p:defrule expression or-expr)
 
 (p:defrule or-expr (and and-expr (* (and spaces? or-op spaces? and-expr)))
   (:function do-binop))
 
 (p:defrule and-expr (and eq-expr (* (and spaces? and-op spaces? eq-expr)))
   (:function do-binop))
 
 (p:defrule eq-expr (and sum-expr (* (and spaces? (or "<=" ">=" "=<" "=>" "<>"
                                                      "=" "<" ">" "!")
                                          spaces? sum-expr)))
   (:function do-binop))
 
 (p:defrule sum-expr (and mod-expr (* (and spaces? (or #\+ #\-) spaces? mod-expr)))
   (:function do-binop))
 
 (p:defrule mod-expr (and mul-expr (* (and spaces? (p:~ "mod") spaces? mul-expr)))
   (:function do-binop))
 
 (p:defrule mul-expr (and unary-expr (* (and spaces? (or #\* #\/) spaces? unary-expr)))
   (:function do-binop))
 
 (p:defrule unary-expr (and (p:? (or #\+ #\-)) atom-expr)
   (:lambda (list)
     (let ((expr (remove-nil list)))
       (if (= 1 (length expr))
           (first expr)
           (intern-first expr)))))
 
 (p:defrule atom-expr (and (or variable literal returning-intrinsic paren-expr) spaces?)
   (:function first))
 
 (p:defrule paren-expr (and #\( spaces? expression spaces? #\))
   (:function third))
 
 (p:defrule or-op (p:~ "or")
   (:constant "or"))
 
 (p:defrule and-op (p:~ "and")
   (:constant "and"))
 
 ;;; Variables
 
 (p:defrule variable (and identifier (p:? array-index))
   (:destructure (id idx-raw)
     (let ((idx (case idx-raw
                  ((nil) 0)
                  (:last nil)
                  (t idx-raw))))
       (list 'lib:qspvar id idx))))
 
 (p:defrule array-index (and #\[ spaces? (p:? expression) spaces? #\])
   (:lambda (list)
     (or (third list) :last)))
 
 (p:defrule assignment (or kw-assignment plain-assignment op-assignment)
   (:destructure (qspvar eq expr)
     (declare (ignore eq))
     (list 'lib:set qspvar expr)))
 
 (p:defrule kw-assignment (and (or (p:~ "let") (p:~ "set")) spaces? (or op-assignment plain-assignment))
   (:function third))
 
 (p:defrule op-assignment (and variable spaces? (or "+" "-" "*" "/") #\= spaces? expression)
   (:destructure (qspvar ws1 op eq ws2 expr)
                 (declare (ignore ws1 ws2))
                 (list qspvar eq (intern-first (list op qspvar expr)))))
 
 (p:defrule plain-assignment (and variable spaces? #\= spaces? expression)
   (:function remove-nil))
 
 ;;; Non-string literals
 
 (p:defrule literal (or qsp-string brace-string number))
 
 (p:defrule number (+ (or #\0 #\1 #\2 #\3 #\4 #\5 #\6 #\7 #\8 #\9))
   (:lambda (list)
     (parse-integer (p:text list))))

src/patches.lisp

@@ -1,131 +1,136 b''
+
+;;;; Parenscript
 
 (in-package parenscript)
 
 ;;; async/await
 
 (defprinter ps-js::await (x)
   (psw (string-downcase "await "))
   (print-op-argument 'ps-js::await x))
 
 (define-trivial-special-ops await ps-js::await)
 
 (define-statement-operator async-defun (name lambda-list &rest body)
   (multiple-value-bind (effective-args body-block docstring)
       (compile-named-function-body name lambda-list body)
     (list 'ps-js::async-defun name effective-args docstring body-block)))
 
 (defprinter ps-js::async-defun (name args docstring body-block)
   (when docstring (print-comment docstring))
   (psw "async ")
   (print-fun-def name args body-block))
 
 (define-expression-operator async-lambda (lambda-list &rest body)
   (multiple-value-bind (effective-args effective-body)
       (parse-extended-function lambda-list body)
     `(ps-js::async-lambda
       ,effective-args
       ,(let ((*function-block-names* ()))
          (compile-function-body effective-args effective-body)))))
 
 (defprinter ps-js::async-lambda (args body-block)
   (psw "async ")
   (print-fun-def nil args body-block))
 
 (cl:export 'await)
 (cl:export 'async-defun)
 (cl:export 'async-lambda)
 
 ;;; ES6
 
 (define-expression-operator => (lambda-list &rest body)
   (unless (listp lambda-list)
     (setf lambda-list (list lambda-list)))
   (multiple-value-bind (effective-args effective-body)
       (parse-extended-function lambda-list body)
     `(ps-js::=>
       ,effective-args
       ,(let ((*function-block-names* ()))
          (compile-function-body effective-args effective-body)))))
 
 (defprinter ps-js::=> (args body)
   (unless (= 1 (length args))
     (psw "("))
   (loop for (arg . remaining) on args do
        (psw (symbol-to-js-string arg)) (when remaining (psw ", ")))
   (unless (= 1 (length args))
     (psw ")"))
   (psw " => ")
   (ps-print body))
 
 (cl:export '=>)
 
 ;;; Actually return nothing (with no empty return)
 (defvar *old-return-result-of* (function return-result-of))
 
 (defun return-result-of (tag form)
   (if (equal form '(void))
       nil
       (funcall *old-return-result-of* tag form)))
 (cl:export 'void)
 
 ;;; Bitwise stuff
 ;; No idea why these are not exported
 (export '<<)
 (export '>>)
 
+;;;; Esrap
+
 (in-package esrap)
+
 (defmethod print-object :around ((condition esrap-error) stream)
   (when (not txt2web::*delivered*)
     (return-from print-object
       (call-next-method)))
   (when (or *print-escape*
             *print-readably*
             (and *print-lines* (<= *print-lines* 5)))
     (return-from print-object))
 
   ;; FIXME: this looks like it won't do the right thing when used as
   ;; part of a logical block.
   (if-let ((text (esrap-error-text condition))
            (position (esrap-error-position condition)))
     (labels ((safe-index (index)
                (min (max index 0) (length text)))
              (find-newline (&key (start 0) (end (length text)) (from-end t))
                (let ((start (safe-index start))
                      (end   (safe-index end)))
                  (cond
                    ((when-let ((position (position #\Newline text
                                                    :start start :end end
                                                    :from-end from-end)))
                       (1+ position)))
                    ((and from-end (zerop start))
                     start)
                    ((and (not from-end) (= end (length text)))
                     end)))))
       ;; FIXME: magic numbers
       (let* ((line       (count #\Newline text :end position))
              (column     (- position (or (find-newline :end position) 0) 1))
              (min-start  (- position 160))
              (max-end    (+ position 24))
              (line-start (or (find-newline :start min-start
                                            :end   position)
                              (safe-index min-start)))
              (start      (cond
                            ((= (safe-index min-start) line-start)
                             line-start)
                            ((find-newline :start min-start
                                           :end   (1- line-start)))
                            (t
 
                             line-start)))
              (end        (or (find-newline :start    position
                                            :end      max-end
                                            :from-end nil)
                              (safe-index max-end)))
              (*print-circle* nil))
         (txt2web::lformat stream :error
                 (= position (length text))
                 (list (subseq text start end))
                 (- position line-start)
                 (1+ line) (1+ column) position)))
 
     (format stream "~2&<text and position not available>~2%")))

src/utils.lisp

@@ -1,26 +1,51 b''
 
 (in-package txt2web)
 
 (defvar *delivered* nil)
 
 (defun src-file (filename)
   (uiop/pathname:merge-pathnames*
    filename
    (asdf:system-source-directory :txt2web)))
 
 (defun read-progn-from-string (string)
   `(progn
      ,@(read-code-from-string string)))
 
 (defun read-code-from-string (string)
   (with-input-from-string (in string)
     (let ((*package* *package*))
       (loop :for form := (read in nil :eof)
          :until (eq form :eof)
          :when (eq (first form) 'cl:in-package)
          :do (setf *package* (find-package (second form)))
          :else
          :collect form))))
 
 (defun load-src (filename)
-  (alexandria:read-file-into-string (src-file filename)))
+  (alexandria:read-file-into-string (src-file filename) :external-format :utf-8))
+
+;;;; For testing
+
+(defvar *dont-expand* '(setf))
+
+(defun should-expand (form)
+  (cond ((not (listp form))
+         nil)
+        ((listp (car form))
+         t)
+        ((member (car form) *dont-expand*)
+         nil)
+        ((not (symbolp (car form)))
+         nil)
+        ((not (eq (symbol-package (car form))
+                  (find-package :parenscript)))
+         t)))
+
+(defun ps-macroexpand-all (form)
+  (if (should-expand form)
+      (let ((form (ps::ps-macroexpand form)))
+        (if (listp form)
+            (mapcar #'ps-macroexpand-all form)
+            form))
+      form))