[Help-smalltalk] [PATCH] allow building documentation for non-built packages

[Paolo Bonzini]

This is bug 103, for which Stephen had a much simpler patch.  But fixing 
it "the right way" (Stephen's patch just avoided breaking the build, but 
left the documentation unbuilt) also showed more use cases for 
gst-package and gst-doc that are worth implementing.

In particular, I added a "--load" option to "gst-package --list-files" 
(would be best to spell it --loaded, actually), which only prints 
filed-in files.  And I changed the default `target directory' for 
--list-files to the current directory (instead of the image file, which 
worked by chance previously because we only invoked gst-package from the 
image directory).

I'll let it stagnate on "master" for a while before moving it to the 
stable branch.  You're invited to test it in the meanwhile.

Paolo

ps: thanks to git, the history for this patch is split in half a dozen 
patches which logically separate the changes.  With the old system it 
would have been way too slow to do so.

pps: I just found the equivalent for "tla missing -s <BRANCH>", that's 
"git cherry -v HEAD <BRANCH>".
2008-01-18  Paolo Bonzini  <address@hidden>
 
        * scripts/Package.st: Change default -t value for --list-files,
        reject --destdir and -t for --list-files.

        * scripts/GenDoc.st: Add --namespace, use #fullyDefinedLoadedClasses.

        * scripts/Package.st: Support multiple --list-files options, add
        --load and --test suboptions to --list-files.
 
        * GSTParser.st: Don't add an existing namespace when parsing
        "Namespace current: ... [ ]".

        * STLoader.st: Add #fullyDefinedLoadedClasses.
        * STLoaderObjs.st: Add #fullyDefinedLoadedClasses.

diff --git a/doc/Makefile.am b/doc/Makefile.am
index 00f0343..403b2e6 100644
--- a/doc/Makefile.am
+++ b/doc/Makefile.am
@@ -26,6 +26,7 @@ MOSTLYCLEANFILES = gst-libs.me gst-libs.mes gst-base.me 
gst-base.mes \
 
 GST_TOOL_ARGS = -I $(top_builddir)/gst.im --kernel-dir $(top_srcdir)/kernel
 GST_DOC = $(top_builddir)/gst-tool gst-doc $(GST_TOOL_ARGS)
+GST_PACKAGE = $(top_builddir)/gst-tool gst-package $(GST_TOOL_ARGS)
 
 ####################################################
 ##
@@ -82,37 +83,58 @@ install-data-local: install-man
 PUBLISHED_CLASSES = Smalltalk.* SystemExceptions.* NetClients.* VFS.*
 
 $(srcdir)/blox.texi: $(top_srcdir)/packages/blox/tk/stamp-classes
-       $(GST_DOC) -p BloxTK -o $(srcdir)/blox.texi BLOX.* || \
+       $(GST_PACKAGE) $(patsubst %, %/package.xml, $(^D)) \
+         --load --list-files=BloxTK | \
+         sed s,^,-f, | \
+         xargs $(GST_DOC) -n BLOX -o $(srcdir)/blox.texi BLOX.* || \
          rm -f $(srcdir)/blox.texi
        test -f $(srcdir)/blox.texi && touch $(srcdir)/gst-libs.texi 
 
 $(srcdir)/tcp.texi: $(top_srcdir)/packages/tcp/stamp-classes
-       $(GST_DOC) -p TCP -o $(srcdir)/tcp.texi TCP.* || \
+       $(GST_PACKAGE) $(patsubst %, %/package.xml, $(^D)) \
+         --load --list-files=TCP | \
+         sed s,^,-f, | \
+         xargs $(GST_DOC) -n TCP -o $(srcdir)/tcp.texi TCP.* || \
          rm -f $(srcdir)/tcp.texi
        test -f $(srcdir)/tcp.texi && touch $(srcdir)/gst-libs.texi 
 
 $(srcdir)/complex.texi: $(top_srcdir)/packages/complex/stamp-classes
-       $(GST_DOC) -p Complex -o $(srcdir)/complex.texi Complex || \
+       $(GST_PACKAGE) $(patsubst %, %/package.xml, $(^D)) \
+         --load --list-files=Complex | \
+         sed s,^,-f, | \
+         xargs $(GST_DOC) -o $(srcdir)/complex.texi Complex || \
          rm -f $(srcdir)/complex.texi
        test -f $(srcdir)/complex.texi && touch $(srcdir)/gst-libs.texi 
 
 $(srcdir)/debug.texi: $(top_srcdir)/packages/debug/stamp-classes
-       $(GST_DOC) -p DebugTools -o $(srcdir)/debug.texi Debugger* || \
+       $(GST_PACKAGE) $(patsubst %, %/package.xml, $(^D)) \
+         --load --list-files=DebugTools | \
+         sed s,^,-f, | \
+         xargs $(GST_DOC) -o $(srcdir)/debug.texi Debugger* || \
          rm -f $(srcdir)/debug.texi
        test -f $(srcdir)/debug.texi && touch $(srcdir)/gst-libs.texi 
 
-$(srcdir)/zlib.texi: $(top_srcdir)/packages/dbi/stamp-classes
-       $(GST_DOC) -p ZLib -o $(srcdir)/zlib.texi ZLib.* || \
+$(srcdir)/zlib.texi: $(top_srcdir)/packages/zlib/stamp-classes
+       $(GST_PACKAGE) $(patsubst %, %/package.xml, $(^D)) \
+         --load --list-files=ZLib | \
+         sed s,^,-f, | \
+         xargs $(GST_DOC) -n ZLib -o $(srcdir)/zlib.texi ZLib.* || \
          rm -f $(srcdir)/zlib.texi
        test -f $(srcdir)/zlib.texi && touch $(srcdir)/gst-libs.texi 
 
 $(srcdir)/dbi.texi: $(top_srcdir)/packages/dbi/stamp-classes
-       $(GST_DOC) -p DBI -o $(srcdir)/dbi.texi DBI.* || \
+       $(GST_PACKAGE) $(patsubst %, %/package.xml, $(^D)) \
+         --load --list-files=DBI | \
+         sed s,^,-f, | \
+         xargs $(GST_DOC) -n DBI -o $(srcdir)/dbi.texi DBI.* || \
          rm -f $(srcdir)/dbi.texi
        test -f $(srcdir)/dbi.texi && touch $(srcdir)/gst-libs.texi 
 
 $(srcdir)/i18n.texi: $(top_srcdir)/packages/iconv/stamp-classes 
$(top_srcdir)/packages/i18n/stamp-classes
-       $(GST_DOC) -p Iconv -p I18N -o $(srcdir)/i18n.texi I18N.* || \
+       $(GST_PACKAGE) $(patsubst %, %/package.xml, $(^D)) \
+         --load --list-files=Iconv --list-files=I18N | \
+         sed s,^,-f, | \
+         xargs $(GST_DOC) -n I18N -o $(srcdir)/i18n.texi I18N.* || \
          rm -f $(srcdir)/i18n.texi
        test -f $(srcdir)/i18n.texi && touch $(srcdir)/gst-libs.texi 
 
diff --git a/gst-tool.c b/gst-tool.c
index 78d5022..542e614 100644
--- a/gst-tool.c
+++ b/gst-tool.c
@@ -117,7 +117,7 @@ struct tool tools[] = {
   {
     "gst-doc", "scripts/GenDoc.st",
     "-h|--help --version -p|--package: -f|--file: -I|--image-file: \
-        -o|--output: --kernel-directory:",
+        -n|--namespace: -o|--output: --kernel-directory:",
     NULL
   },
   { NULL, NULL, NULL, NULL }
diff --git a/packages/stinst/parser/GSTParser.st 
b/packages/stinst/parser/GSTParser.st
index 5a1a045..a452eb0 100644
--- a/packages/stinst/parser/GSTParser.st
+++ b/packages/stinst/parser/GSTParser.st
@@ -8,7 +8,7 @@
 
 "======================================================================
 |
-| Copyright 2007 Free Software Foundation, Inc.
+| Copyright 2007, 2008 Free Software Foundation, Inc.
 | Written by Daniele Sciascia.
 |
 | This file is part of the GNU Smalltalk class library.
@@ -111,15 +111,18 @@ STInST.STFileInParser subclass: GSTFileInParser [
     ]
     
     parseNamespace: node [   
-        | namespace fullNamespace |
+        | namespace fullNamespace newNamespace |
         namespace := RBVariableNode
            named: self driver currentNamespace name asString.
         fullNamespace := RBVariableNode
            named: (self driver currentNamespace nameIn: Smalltalk).
 
-        self evaluateMessageOn: namespace
-             selector: #addSubspace:
-             argument: node arguments first name asSymbol.
+       newNamespace := node arguments first name asSymbol.
+       (self driver currentNamespace includesKey: newNamespace)
+           ifFalse: [
+               self evaluateMessageOn: namespace
+                    selector: #addSubspace:
+                    argument: node arguments first name asSymbol ].
            
         self evaluateStatement: node.       
         self parseDoits.
diff --git a/packages/stinst/parser/STLoader.st 
b/packages/stinst/parser/STLoader.st
index 5557ff5..4488a12 100644
--- a/packages/stinst/parser/STLoader.st
+++ b/packages/stinst/parser/STLoader.st
@@ -8,7 +8,7 @@
 
 "======================================================================
 |
-| Copyright 2001, 2002, 2007 Free Software Foundation, Inc.
+| Copyright 2001, 2002, 2007, 2008 Free Software Foundation, Inc.
 | Written by Paolo Bonzini.
 |
 | This file is part of GNU Smalltalk.
@@ -231,6 +231,10 @@ loadedClasses
     ^loadedClasses
 !
 
+fullyDefinedLoadedClasses
+    ^loadedClasses select: [ :each | each isFullyDefined ]
+!
+
 result
     "This is what #parseSmalltalk answers"
     ^self loadedClasses
diff --git a/packages/stinst/parser/STLoaderObjs.st 
b/packages/stinst/parser/STLoaderObjs.st
index d48a624..d02cb05 100644
--- a/packages/stinst/parser/STLoaderObjs.st
+++ b/packages/stinst/parser/STLoaderObjs.st
@@ -8,7 +8,7 @@
 
 "======================================================================
 |
-| Copyright 1999, 2000, 2001, 2002, 2007 Free Software Foundation, Inc.
+| Copyright 1999, 2000, 2001, 2002, 2007, 2008 Free Software Foundation, Inc.
 | Written by Paolo Bonzini.
 |
 | This file is part of GNU Smalltalk.
@@ -557,6 +557,11 @@ isDefined
     ^true
 !
 
+isFullyDefined
+    self isDefined ifFalse: [ ^false ].
+    ^self superclass isNil or: [ self superclass isFullyDefined ]
+! !
+
 
 !PseudoBehavior methodsFor: 'abstract'!
 
@@ -655,6 +660,16 @@ on: aClass for: aSTClassLoader
     ^(self for: aSTClassLoader) setProxy: aClass
 ! !
 
+!ProxyClass methodsFor: 'testing'!
+
+isDefined
+     ^true
+!
+
+isFullyDefined
+     ^true
+! !
+
 !ProxyClass methodsFor: 'delegation'!
 
 = anObject
diff --git a/scripts/GenDoc.st b/scripts/GenDoc.st
index a79af18..b5e2b78 100644
--- a/scripts/GenDoc.st
+++ b/scripts/GenDoc.st
@@ -31,19 +31,21 @@
 "Load the prerequisites"
 PackageLoader fileInPackage: #ClassPublisher!
 
-| package outFile publisher files classes classPatterns loader |
+| package outFile publisher files classes classPatterns loader 
defaultNamespace |
 
 classPatterns := OrderedCollection new.
 publisher := STInST.TexinfoDocPublisher.
+defaultNamespace := Smalltalk.
 
 helpString :=
 'Usage:
     gst-doc [ flag ... ] class ...
 
 Options:
+    -I --image-file=FILE  look for classes in the given image
     -p --package=PKG      look for classes in the given package
     -f --file=FILE        look for classes in the given file
-    -I --image-file=FILE  look for classes in the given image
+    -n --namespace=NAMESP load files in the given namespace
     -o --output=FILE      emit documentation in the given file (default=stdout)
        --kernel-dir=PATH  use the specified kernel directory
     -h --help             show this message
@@ -56,7 +58,7 @@ Options:
 
     "Parse the command-line arguments."
     Smalltalk
-        arguments: '-h|--help --version -p|--package: -f|--file:
+        arguments: '-h|--help --version -p|--package: -f|--file: 
-n|--namespace:
                     -I|--image-file: -o|--output: --kernel-directory:'
         do: [ :opt :arg |
 
@@ -73,6 +75,17 @@ Options:
                self error: '--output specified multiple times' ].
            outFile := arg ].
 
+       opt = 'namespace' ifTrue: [
+            defaultNamespace := Smalltalk.
+            (arg subStrings: $.) do:
+                [:each |  
+                | key |
+                key := each asSymbol.
+                (defaultNamespace includesKey: key)
+                   ifFalse: [defaultNamespace addSubspace: key].
+                defaultNamespace := defaultNamespace at: key].
+               loader currentNamespace: defaultNamespace ].
+
         opt = 'package' ifTrue: [
            package := PackageLoader packageAt: arg.
            loader currentNamespace: package createNamespace.
@@ -83,7 +96,7 @@ Options:
                    parseSmalltalkStream: file
                    with: STInST.GSTFileInParser ].
 
-           loader currentNamespace: Smalltalk ].
+           loader currentNamespace: defaultNamespace ].
 
         opt = 'file' ifTrue: [
            file := FileStream open: arg mode: FileStream read.
@@ -99,12 +112,12 @@ Options:
 
     classPatterns isEmpty
         ifTrue: [
-            classes := loader loadedClasses.
+            classes := loader fullyDefinedLoadedClasses.
             classes isEmpty
                ifTrue: [ self error: 'specify -p, -f, or a class name' ] ]
         ifFalse: [
            allClasses :=
-               loader loadedClasses,
+               loader fullyDefinedLoadedClasses,
                (Class allSubclasses collect: [ :each | each instanceClass ]).
 
            classes := IdentitySet new.
diff --git a/scripts/Package.st b/scripts/Package.st
index d02b53f..be352a3 100644
--- a/scripts/Package.st
+++ b/scripts/Package.st
@@ -267,14 +267,23 @@ Object subclass: Command [
     runOnPackage: aPackage [ ]
 
     listFiles: listFiles [
-       | base vpathBase vpath |
+       | base vpathBase vpath source test |
        vpath := self isOption: 'vpath'.
+       source := self isOption: 'load'.
+       test := self isOption: 'test'.
        base := self installDir.
        vpathBase := Directory name: self srcdir.
 
-        listFiles do: [ :each || package |
+        listFiles do: [ :each || package files |
            package := self packages at: each.
-           package allFiles do: [ :file |
+           files := source
+               ifFalse: [ package allFiles ]
+               ifTrue: [ package fileIns ].
+
+           (test and: [ source and: [ package test notNil ]])
+               ifTrue: [ files := files, package test fileIns ].
+
+           files do: [ :file |
                | path relativePath |
                path := package fullPathOf: file.
                relativePath := base pathTo: path.
@@ -474,8 +483,14 @@ Command subclass: PkgUninstall [
 ]
 
 Command subclass: PkgList [
-    validOptions [ ^#('list-files' 'vpath') ]
-    validateDestDir: destdir installDir: installDir [ ]
+    validOptions [ ^#('list-files' 'vpath' 'load') ]
+    validateDestDir: destdir installDir: installDir [
+       destdir = ''
+           ifFalse: [ self error: '--destdir not needed with --list-files' ].
+       installDir isNil
+           ifFalse: [ self error: '--target-directory not needed with 
--list-files' ]
+    ]
+    defaultInstallDir [ ^'.' ]
 ]
 
 PkgList subclass: PkgPackageList [
@@ -682,6 +697,8 @@ Common suboptions:
        --load                  also load the Smalltalk files in the image
 
 --list-files suboptions:
+       --load                  only list files that are filed in when loading
+       --test                  with --load, also include unit test files
        --vpath                 Omit path to srcdir for files that are there
 
 --dist suboptions:
@@ -730,7 +747,8 @@ The default target directory is ', Directory image, '
                mode isNil ifFalse: [ self error: 'multiple modes specified' ].
                mode := PkgPrepare ].
             opt = 'list-files' ifTrue: [
-               mode isNil ifFalse: [ self error: 'multiple modes specified' ].
+               (mode isNil or: [ mode = PkgList ])
+                    ifFalse: [ self error: 'multiple modes specified' ].
                listFiles add: arg.
                mode := PkgList ].