Initial commit.

Author: xach

Makefile

@@ -0,0 +1,15 @@
+all: system-file-magic depcheck
+
+depcheck: asdf.lisp depcheck.lisp
+	buildapp --dynamic-space-size 4000 --output depcheck --load asdf.lisp --require sb-aclrepl  --require sb-bsd-sockets --require sb-rt --require sb-cover --require sb-sprof --entry depcheck:main --require sb-introspect --require sb-cltl2 --require sb-posix --require sb-concurrency --load depcheck.lisp
+
+
+system-file-magic: asdf.lisp system-file-magic.lisp
+	buildapp --dynamic-space-size 4000 --output system-file-magic --load asdf.lisp --require sb-posix --load system-file-magic.lisp --entry system-file-magic:main
+
+install: system-file-magic depcheck
+	install -c -m 555 system-file-magic $(HOME)/bin
+	install -c -m 555 depcheck $(HOME)/bin
+
+clean:
+	rm -f system-file-magic depcheck

README

@@ -0,0 +1,28 @@
+The Quicklisp Controller is responsible for building and updating
+Quicklisp distributions from a project list (maintained separately in
+the quicklisp-projects repo).
+
+It fetches project source from a variety of places: git, cvs, svn,
+http-available tarballs, etc, and uses that to build up metadata about
+the relationships between projects. New or updated projects are then
+uploaded into the quicklisp download space.
+
+You must create ~/quicklisp-controller/ and symlink a checkout of
+quicklisp-projects (or something with similar directory structure) to
+~/quicklisp-controller/projects. A quick way to do that:
+
+  (setup-directories "~/path/to/quicklisp-projects/")
+
+To fetch/update all project sources, use:
+
+  (update-what-you-can)
+
+To build everything that can be built, use:
+
+  (ensure-what-wins-you-can)
+
+To combine those two operations, concluding with a report of what
+differs from the last Quicklisp dist:
+
+  (recrank)
+

asdf.lisp

@@ -0,0 +1,50 @@
+;;;; client-uploader.lisp
+
+(in-package #:quicklisp-controller)
+
+(defparameter *quickstart-bucket* "beta.quicklisp.org")
+
+(defun invalidate-quickstart-paths (paths)
+  (let ((distributions (zs3:distributions-for-bucket *quickstart-bucket*))
+        (paths (mapcar (lambda (path)
+                         (if (char= (char path 0) #\/)
+                             path
+                             (format nil "/~A" path)))
+                       paths)))
+    (dolist (distribution distributions)
+      (zs3:invalidate-paths distribution paths))))
+
+(defun upload-client-files ()
+  (with-posix-cwd "~/src/quicklisp-client/"
+    (run "make")
+    (let* ((version (first-line-of "version.txt"))
+           (versioned-key (format nil "quickstart/quicklisp-~A.tgz" version)))
+      (zs3:put-file "quicklisp.tar"
+                    *quickstart-bucket* "quickstart/quicklisp.tar"
+                    :public t)
+      (zs3:put-file (file-namestring versioned-key)
+                    *quickstart-bucket* versioned-key
+                    :public t)
+      (zs3:put-file "version.txt" *quickstart-bucket* "quickstart/version.txt"
+                    :public t :content-type "text/plain")
+      (invalidate-quickstart-paths (list "/quickstart/version.txt"
+                                         "/quickstart/quicklisp.tar"
+                                         versioned-key)))))
+
+(defun upload-bootstrap-files ()
+  (with-posix-cwd "~/src/quicklisp-bootstrap/"
+    (zs3:put-file "asdf.lisp"
+                  *quickstart-bucket*
+                  "quickstart/asdf.lisp"
+                  :public t
+                  :content-type "text/plain")
+    (zs3:put-file "quicklisp.lisp" *quickstart-bucket* "quicklisp.lisp"
+                  :public t
+                  :content-type "text/plain"
+                  :content-disposition "attachment")
+    (zs3:put-file "setup.lisp" *quickstart-bucket* "quickstart/setup.lisp"
+                  :public t
+                  :content-type "text/plain")
+    (invalidate-quickstart-paths '("/quicklisp.lisp"
+                                   "/quickstart/asdf.lisp"
+                                   "/quickstart/setup.lisp"))))

client-uploader.lisp

@@ -0,0 +1,87 @@
+;;;; commands.lisp
+
+(in-package #:quicklisp-controller)
+
+(defvar *command-output* (make-synonym-stream '*standard-output*))
+
+(define-condition run-error (error)
+  ((command
+    :initarg :command
+    :reader run-error-command)
+   (arguments
+    :initarg :arguments
+    :reader run-error-arguments)
+   (exit-code
+    :initarg :exit-code
+    :reader run-error-exit-code))
+  (:report (lambda (condition stream)
+             (format stream "Run error (exit code ~D) for:~%  ~S ~{~S~^ ~}"
+                     (run-error-exit-code condition)
+                     (run-error-command condition)
+                     (run-error-arguments condition)))))
+
+(defun stringify-command-argument (argument)
+  (typecase argument
+    (null nil)
+    (string argument)
+    (pathname (native-namestring argument))
+    (keyword (format nil "--~(~A~)" argument))
+    (t (princ-to-string argument))))
+
+(defun run (command &rest arguments)
+  (let* ((arguments (remove nil
+                            (mapcar #'stringify-command-argument arguments)))
+         (process (run-program command arguments
+                               :search t
+                               :wait t
+                               :output *command-output*)))
+    (unwind-protect
+         (let ((code (process-exit-code process)))
+           (if (zerop code)
+               t
+               (error 'run-error
+                      :exit-code code
+                      :command command
+                      :arguments arguments)))
+      (ignore-errors (process-close process)))))
+
+(defmacro with-run-output ((stream (command &rest args)) &body body)
+  `(let* ((*command-output* (make-string-output-stream)))
+     (run ,command ,@args)
+     (with-input-from-string (,stream (get-output-stream-string  *command-output*))
+       ,@body)))
+
+(defun native-directory-string (pathname)
+  (native-namestring (directory-namestring (probe-file pathname))))
+
+(defmacro with-posix-cwd (new-directory &body body)
+  ;; fchdir thing from Linux's getcwd(3)
+  (let ((fd (gensym))
+        (new (gensym)))
+    `(let ((,fd nil)
+           (,new (native-directory-string ,new-directory)))
+       (unwind-protect
+            (let ((*default-pathname-defaults* (probe-file ,new)))
+              (setf ,fd (sb-posix:open "." 0))
+              (sb-posix:chdir ,new)
+              ,@body)
+         (when ,fd
+           (sb-posix:fchdir ,fd)
+           (ignore-errors (sb-posix:close ,fd)))))))
+
+(defmacro with-binary-run-output (pathname &body body)
+  `(with-open-file (*command-output* ,pathname :direction :output
+                                     :element-type '(unsigned-byte 8)
+                                     :if-exists :supersede)
+     ,@body))
+
+(defmacro without-run-output (&body body)
+  `(let ((*command-output* nil))
+     ,@body))
+
+(defun run-output-lines (command &rest args)
+  (let ((output (with-output-to-string (*command-output*)
+                  (apply #'run command args))))
+    (with-input-from-string (stream output)
+      (loop for line = (read-line stream nil)
+            while line collect line))))

commands.lisp

@@ -0,0 +1,6 @@
+;;;; config.lisp
+
+(in-package #:quicklisp-controller)
+
+(defvar *report-to-email* nil
+  "The email address to which reports are emailed.")

config.lisp

@@ -0,0 +1,135 @@
+;;; depcheck.lisp
+
+(defpackage #:depcheck
+  (:use #:cl))
+
+(in-package #:depcheck)
+
+(defvar *direct-dependencies* nil)
+
+(defun load-asdf-system-table (file)
+  (let ((table (make-hash-table :test 'equalp)))
+    (with-open-file (stream file)
+      (loop for line = (read-line stream nil)
+            while line do
+            (let ((pathname
+                   (merge-pathnames line
+                                    file)))
+              (setf (gethash (pathname-name pathname) table)
+                    (truename pathname)))))
+    table))
+
+(defvar *systems* nil)
+
+(defun real-system-name (name)
+  (setf name (string name))
+  (subseq name 0 (position #\/ name)))
+
+(defun system-finder (name)
+  (when *systems*
+    (gethash (real-system-name name) *systems*)))
+
+(defun sbcl-contrib-p (name)
+  (and (<= 3 (length name))
+       (string= name "sb-" :end1 3)))
+
+(defun normalize-dependency (name)
+  (cond ((and (consp name)
+              (keywordp (first name)))
+         (string-downcase (second name)))
+        ((or (symbolp name) (stringp name))
+         (string-downcase name))
+        (t (error "Don't know how to normalize ~S" name))))
+
+(defun make-hook (old-hook system-name)
+  (lambda (fun form env)
+    (when (and (consp form)
+               (eq (first form) 'asdf:defsystem)
+               (string-equal (second form) system-name))
+      (let ((deps (getf (cddr form) :depends-on))
+            (prereqs (getf (cddr form) :defsystem-depends-on))
+            (weak (getf (cddr form) :weakly-depends-on)))
+        (setf deps (append deps prereqs weak))
+        (setf *direct-dependencies* (mapcar 'normalize-dependency deps))))
+    (funcall old-hook fun form env)))
+
+(defvar *in-find-system* nil)
+(defvar *implied-dependencies* nil)
+
+(defvar *load-op-wrapper*
+  '(defmethod asdf:operate :around ((op (eql 'asdf:load-op)) system
+                                    &key &allow-other-keys)
+    (cond (*in-find-system*
+           (push (asdf::coerce-name system) *implied-dependencies*)
+           (let ((*in-find-system* nil))
+             (call-next-method)))
+          (t
+           (call-next-method)))))
+
+(defvar *metadata-required-p* nil)
+
+(defun check-system-metadata (system)
+  (when *metadata-required-p*
+    (flet ((check-attribute (fun description)
+             (let ((value (funcall fun system)))
+               (cond ((not value)
+                      (error "Missing ~A for system ~A"
+                             description
+                             (asdf:component-name system)))
+                     ((and (stringp value) (zerop (length value)))
+                      (error "Empty ~A for system ~A"
+                             description
+                             (asdf:component-name system))))
+               (when (and (stringp value) (zerop (length value)))))))
+      (check-attribute 'asdf:system-description :description)
+      ;; Not yet
+      ;;(check-attribute 'asdf:system-license :license)
+      (check-attribute 'asdf:system-author :author))))
+
+(defun compute-dependencies (system-file system-name)
+  (let* ((asdf:*system-definition-search-functions*
+          (list #-asdf3 'asdf::sysdef-find-asdf
+                'system-finder))
+         (dependencies nil)
+         (*direct-dependencies* nil)
+         (*macroexpand-hook* (make-hook *macroexpand-hook* system-name)))
+    (let ((*implied-dependencies* nil)
+          (*in-find-system* t))
+      (check-system-metadata (asdf:find-system system-file))
+      (setf dependencies *implied-dependencies*))
+    (asdf:oos 'asdf:load-op system-name)
+    (setf dependencies
+          (remove-duplicates (append *direct-dependencies* dependencies)
+                             :test #'equalp))
+    (sort (remove-if #'sbcl-contrib-p dependencies) #'string<)))
+
+(defun magic (system-file system trace-file)
+  (handler-bind ((sb-ext:defconstant-uneql #'continue))
+    (with-open-file (stream trace-file :direction :output
+                            :if-exists :supersede)
+      (format stream "~A~{ ~A~}~%"
+              system (compute-dependencies system-file system)))))
+
+(defun main (argv)
+  (setf *print-pretty* nil)
+  (sb-posix:setenv "SBCL_HOME"
+                   (load-time-value
+                    (directory-namestring sb-int::*core-string*))
+                   1)
+  (sb-posix:setenv "CC" "gcc" 1)
+  (eval *load-op-wrapper*)
+  (destructuring-bind (index project system dependency-file errors-file
+                             &optional *metadata-required-p*)
+      (rest argv)
+    (setf *systems* (load-asdf-system-table index))
+    (with-open-file (*error-output* errors-file
+                                    :if-exists :supersede
+                                    :direction :output)
+      (unless (sb-posix:getenv "DEPCHECK_DEBUG")
+        (sb-ext:disable-debugger))
+      (unwind-protect
+           (magic project system dependency-file)
+        (ignore-errors (close *error-output*))))
+    (when (probe-file dependency-file)
+      (delete-file errors-file))))
+

depcheck.lisp

@@ -0,0 +1,42 @@
+;;;; descriptions.lisp
+
+(in-package #:quicklisp-controller)
+
+;;;
+;;; Create a description .cdb file for a mock dist
+;;; based on the dist build descriptions file
+;;;
+
+(defun descriptions-table ()
+  (let ((table (make-string-table)))
+    (with-open-file (stream #p"qlc:descriptions.txt")
+      (loop for (system description) = (read stream nil)
+            while system do
+            (when (plusp (length description))
+              (setf (gethash system table) description))))
+    table))
+
+(defun encode-description (string)
+  (let ((octets (babel:string-to-octets string :encoding :utf-8)))
+    (remove-if (lambda (code)
+                 (< code 32))
+               octets)))
+
+(defun write-dist-descriptions-cdb (dist-name)
+  (let* ((dist (or (ql-dist:dist dist-name)
+                  (error "Unknown dist ~S" dist-name)))
+         (file (ql-dist:relative-to dist "descriptions.cdb"))
+         (tmp (ql-dist:relative-to dist "descriptions.tmp"))
+         (descriptions (descriptions-table)))
+    (zcdb:with-output-to-cdb (cdb file tmp)
+      (dolist (system (ql-dist:provided-systems dist))
+        (let* ((name (ql-dist:name system))
+               (description (gethash name descriptions)))
+          (restart-bind ((skip (lambda (&optional val)
+                                 (declare (ignore val))
+                                 (go :skip))))
+            (when description
+            (zcdb:add-record (ascii-encode name)
+                             (encode-description description)
+                             cdb))))
+        :skip))))