Index: bin/gnump3d-index =================================================================== RCS file: /cvsroot/gnump3d/gnump3d/bin/gnump3d-index,v retrieving revision 1.15 diff -u -r1.15 gnump3d-index --- bin/gnump3d-index 15 Mar 2005 19:14:16 -0000 1.15 +++ bin/gnump3d-index 17 Mar 2005 03:39:12 -0000 @@ -38,8 +38,8 @@ # On a Dual Athlon (1.6 GHz each) and 600+ songs in 60 directories, this # indexing was taking 20 minutes!?! Maybe make the indexing a bit smarter, # look to see if we have the data already. Look first for pathname, then -# look at file modification time (like Make). So now tag is: -# /path/to/file\tmtime\tLENGTH=.*\t... +# look at file modification time (like Make). mtime is stored in the tag +# list just as any other tag. # Also, I suppose we can force user to store files with proper "extensions" # (.ogg, .mp3, ...), but really we want the type of data in the file (as per # the 'file' command), not something from the name of the file. @@ -197,7 +197,8 @@ # Read in existing cache here, if present. &readCache( $cache ) if( -e $cache ); - +# And the update file, if present. +&readCache( "$cache.updates" ) if( -e "$cache.updates" ); # # Do the indexing - following symlinks on non-Windows platforms. @@ -218,6 +219,11 @@ &indexFiles( ); # +# Remove the updates file if present. +# +unlink("$cache.updates") if( -e "$cache.updates" ); + +# # Tidy up. &removeLock(); @@ -225,7 +231,6 @@ # Finished exit; - # # Use the excellent File::Find module to locate all the files beneath # our archive root. @@ -405,12 +410,8 @@ @fstat = stat( $file ); } if( exists( $cache{$file} ) ) { - if( $#fstat == 12 ) { # UNIX stat structure, mtime is index 9 - if( $cache{$file}->{mtime} == $fstat[9] ) { - $skip = 1; - } - } else { - # Unknown, build entry manually. + if( $cache{$file}->{"MTIME"} == $fstat[9] ) { + $skip = 1; } } else { # No cache, build entry manually. @@ -440,16 +441,10 @@ # Make sure filename is defined.. # my $base = $file; - if ( $base =~ /(.*)\/(.*)/ ) - { - # Strip away the leading directory name. - $base = $2; - } - if ( $base =~ /(.*)\.(.*)/ ) - { - # Remove any suffix. - $base = $1; - } + if ( $base =~ /(.*\/)(.*)\.(.*)/ ) { + # Strip away the leading directory name and extension. + $base = $2; + } $TAGS{ 'FILENAME' } = $base; # @@ -463,21 +458,13 @@ } } - if( $#fstat == 12 ) { # UNIX stat structure, mtime is index 9 - $TAGS{mtime} = $fstat[9]; - } + $TAGS{"MTIME"} = $fstat[9]; # # Write the filename and tag details to the cache file, as # tab seperated values. # - if( $TAGS{mtime} ) { - print OUT "$file\tmtime=$TAGS{mtime}"; - delete( $TAGS{mtime} ); - - } else { - print OUT "$file"; - } + print OUT "$file"; foreach my $k ( keys %TAGS ) { @@ -493,13 +480,7 @@ print OUT "\n"; } else { # We have a previously cached entry to write. - if( $cache{$file}->{mtime} ) { - print OUT "$file\tmtime=$cache{$file}->{mtime}"; - delete( $cache{$file}->{mtime} ); - - } else { - print OUT "$file"; - } + print OUT "$file"; foreach my $k ( keys %{$cache{$file}} ) { @@ -784,14 +765,9 @@ print "Error: duplicate entries for ($pathname)\n"; } my $entry = {}; - if( $fields[0] =~ /^mtime=(\d+)$/ ) { # Pure number, assume mtime - $entry->{mtime} = $1; - shift( @fields ); - } else { - $entry->{mtime} = -1; # Old dbase format, no mtime - } foreach my $field (@fields) { my($key,$value,@junk) = split(/=/, $field); + $key = 'MTIME' if ($key eq 'mtime'); # Use empty string for undef. $value = defined( $value ) ? $value : ''; # print "Junk in field ($field)\n" if( @junk ); Index: bin/gnump3d2 =================================================================== RCS file: /cvsroot/gnump3d/gnump3d/bin/gnump3d2,v retrieving revision 1.89 diff -u -r1.89 gnump3d2 --- bin/gnump3d2 16 Mar 2005 14:39:10 -0000 1.89 +++ bin/gnump3d2 17 Mar 2005 03:39:14 -0000 @@ -1180,7 +1180,9 @@ # close($data); } + exit; # terminate the child. + } else { # # Parent from the fork(); # @@ -1190,7 +1192,8 @@ $reload_cache = 0; } - exit; + # then check for updates; + $tagCache->checkForUpdates(); } #nothing to do.. wait for next request @@ -2109,13 +2112,10 @@ } else { @fstat = stat( "$dir/$file" ); } - if( $#fstat == 12 ) { # UNIX stat structure, mtime is index 9 - if (($fstat[9] > $tagCache->getCacheMod()) || - ($fstat[9] > (time() - ($new_days * 86400)))) { - $row =~ s/\$NEW/$new_format/g; - } else { - $row =~ s/\$NEW//g; - } + + if (($fstat[9] > $tagCache->getCacheMod()) || + ($fstat[9] > (time() - ($new_days * 86400)))) { + $row =~ s/\$NEW/$new_format/g; } else { $row =~ s/\$NEW//g; } @@ -3376,7 +3376,7 @@ $path = $1 . $2; } - return( $uri ); + return( $path ); } Index: lib/gnump3d/tagcache.pm =================================================================== RCS file: /cvsroot/gnump3d/gnump3d/lib/gnump3d/tagcache.pm,v retrieving revision 1.12 diff -u -r1.12 tagcache.pm --- lib/gnump3d/tagcache.pm 15 Mar 2005 19:14:16 -0000 1.12 +++ lib/gnump3d/tagcache.pm 17 Mar 2005 03:39:15 -0000 @@ -51,6 +51,7 @@ my $HIDE_TAGS = 0; my $DISABLE_CACHE = 0; my $CACHE_MOD = 0; +my $UPDATE_MOD = 0; my $NEW_DAYS = 0; my %CACHE = ( ); @@ -160,44 +161,75 @@ my $file = shift; my $count = 0; - if ( ( defined( $CACHE_FILE ) ) and - ( $DISABLE_CACHE == 0 )) - { - my @finfo = stat($file); - if (($finfo[9] > $CACHE_MOD) or ($file ne $CACHE_FILE)) - { - open( FILY, "<$file" ) - or die "Cannot read cache file $file - $!"; - - foreach () - { - my %TAGS; - chomp; - $count++; - my @NAMES = split( /\t/, $_); - my $file = shift(@NAMES); - foreach my $pair ( @NAMES ) - { - if ( ( $pair =~ /([A-Z]+)=(.*)/ ) && - ( length( $2 ) ) ) - { - $TAGS{ $1 } = $2; - } - } - $CACHE{$file} = \%TAGS; + if ( ( $DISABLE_CACHE == 0 ) and + ( -e $file ) ) { + my @finfo = stat($file); + if (($finfo[9] > $CACHE_MOD) or ($file ne $CACHE_FILE)) { + open( FILY, "<$file" ) + or die "Cannot read cache file $file - $!"; + + foreach () { + my %TAGS; + chomp; + $count++; + my @NAMES = split( /\t/, $_); + my $file = shift(@NAMES); + foreach my $pair ( @NAMES ) { + if ( ( $pair =~ /([A-Z]+)=(.*)/ ) && + ( length( $2 ) ) ) { + $TAGS{ $1 } = $2; } - close( FILY ); - if ($count > 0) - { -# print "Tag Cache initialized, $count entries\n"; + } + $CACHE{$file} = \%TAGS; + } + close( FILY ); + } + $CACHE_MOD = $finfo[9]; + $CACHE_FILE = $file; + + if (-e "$file.updates") { + @finfo = stat("$file.updates"); + if (($finfo[9] > $UPDATE_MOD) and + ($finfo[9] >= $CACHE_MOD)) { + open( FILY, "<$file.updates" ) + or die "Cannot read cache file $file.updates - $!"; + + foreach () { + my %TAGS; + chomp; + $count++; + my @NAMES = split( /\t/, $_); + my $file = shift(@NAMES); + foreach my $pair ( @NAMES ) { + if ( ( $pair =~ /([A-Z]+)=(.*)/ ) && + ( length( $2 ) ) ) { + $TAGS{ $1 } = $2; + } } + $CACHE{$file} = \%TAGS; + } + close( FILY ); } - $CACHE_MOD = $finfo[9]; + $UPDATE_MOD = $finfo[9]; + } + } + + if ($count > 0) { + # print "Tag Cache initialized, $count entries\n"; } - $CACHE_FILE = $file; } # +# check for (and load) updated cache files. +# +sub checkForUpdates +{ + my $self = shift; + $self->setCacheFile( $CACHE_FILE ); +} + + +# # Obtain and format the song tags for a collection of files, this is # massively faster than doing the same operation on a single file. # @@ -270,23 +302,6 @@ if (exists($CACHE{$file})) { %TAGS = %{$CACHE{$file}}; - - # Convert the file size to something more - # readable - if ( defined( $TAGS{ 'SIZE' } ) ) - { - my $sizeTotal = $TAGS{'SIZE'}; - - $sizeTotal = $sizeTotal < (1024) ? - $sizeTotal . " bytes" : ( - $sizeTotal < (1024 ** 2) ? - (int (10 * $sizeTotal/1024)/10) . "K" : ( - $sizeTotal < (1024 ** 3) ? - (int (10 * $sizeTotal/(1024 ** 2) )/10) . "Mb" : - ((int (10 * $sizeTotal/(1024 ** 3) )/10) . "Gb"))); - - $TAGS{'SIZE'} = $sizeTotal; - } } else { @@ -294,14 +309,10 @@ my $filename = $file; # Just store the filename. - if ( $file =~ /(.*)\/(.*)/ ) + if ( $file =~ /(.*\/)?(.*)\.(.*)/ ) { $filename = $2; } - if ( $file =~ /(.*)\.(.*)/ ) - { - $filename = $1; - } # we'll always have a filename. $TAGS{ 'FILENAME' } = $filename; @@ -313,9 +324,7 @@ } else { @fstat = stat( $file ); } - if( $#fstat == 12 ) { # UNIX stat structure, mtime is index 9 - $TAGS{"mtime"} = $fstat[9]; - } + $TAGS{"MTIME"} = $fstat[9]; # # load up tag info from the file. @@ -384,11 +393,30 @@ } } + # Store the entry in our cache file, if enabled + goto skipped if ( ( -z $CACHE_FILE ) or + ( $DISABLE_CACHE != 0) ); + + open(UPFILE, ">>$CACHE_FILE.updates") || goto skipped; + print UPFILE "$file"; + foreach my $k ( keys(%TAGS) ) { + my $value = $TAGS{ $k }; + + # Replace tabs in tag values with spaces so that the reading + # code doesn't get confused by excessive deliminators. (Curious, + # why 5 spaces? GH) + $value =~ s/\t/ /g; + + print UPFILE "\t" . $k . "=" . $value; + } + print UPFILE "\n"; + close (UPFILE); + + skipped: # And now, store the results in the cache so we don't have # to look it up again. $CACHE{$file} = \%TAGS; - # XXXXX STORE ME IN CACHE AND/OR MARK FOR ADDITION. - } + } # # Work with a copy - we destroy the original @@ -437,17 +465,37 @@ } } - $length = ( $secs * 60 ) + - ( $mins * 60 * 60 ) + + $length = ( $secs * 60 ) + + ( $mins * 60 * 60 ) + ( $hours * 60 * 60 * 60 ); $format = $pre . $length . $post; next; } + if ( $key eq "SIZE" ) { + if ( defined( $TAGS{ 'SIZE' } ) ) { + my $sizeTotal = $TAGS{'SIZE'}; + + $sizeTotal = $sizeTotal < (1024) ? + $sizeTotal . " bytes" : ( + $sizeTotal < (1024 ** 2) ? + (int (10 * $sizeTotal/1024)/10) . "K" : ( + $sizeTotal < (1024 ** 3) ? + (int (10 * $sizeTotal/(1024 ** 2) )/10) . "Mb" : + ((int (10 * $sizeTotal/(1024 ** 3) )/10) . "Gb"))); + + $format = $pre . $sizeTotal . $post; + } else { + $format = $pre . $post; + } + next; + } + if ( $key eq "NEW" ) { - if (exists($TAGS{"mtime"}) && (($TAGS{"mtime"} > $CACHE_MOD) || - ($TAGS{"mtime"} > (time() - ($NEW_DAYS * 86400))))) { +# print "tag_mtime $TAGS{MTIME} cache_mod $CACHE_MOD update_mod $UPDATE_MOD\n"; + if (exists($TAGS{"MTIME"}) && (($TAGS{"MTIME"} > $CACHE_MOD) || + ($TAGS{"MTIME"} > (time() - ($NEW_DAYS * 86400))))) { $format = $pre . $NEW_FORMAT_STRING . $post; } else { $format = $pre . $post; @@ -455,13 +503,10 @@ next; } - if (defined( $TAGS{$key} ) ) - { + if (defined($TAGS{$key})) { # Do the insertion. $format = $pre . gnump3d::url::encodeEntities( $TAGS{$key} ) . $post; - } - else - { + } else { $format = $pre . $post; } } @@ -485,7 +530,7 @@ next if ( $key eq 'UPPER' ); next if ( $key eq 'VERSION' ); next if ( $key eq 'WINDOW' ); - next if ( $key eq 'mtime' ); + next if ( $key eq 'MTIME' ); next if ( $TAGS{"$key"} eq '' ); # tag cache can have "empty" tags. $valid++; @@ -573,12 +618,16 @@ =item C -Specify the name of the cache file to read when formatting tags. If a cache file has previously been specified this will be used instead. +Specify the name of the cache file to read when formatting tags. If a cache file has previously been specified this will be used instead. If the same file is specified, this will check for updates, and if so, load them. =item C Return the name of the cache file we're going to read the tags from. +=item C + +Check for updated cache/updatecache files, and if updated, load 'em up. + =item C Obtain and format the song tags for a collection of files, this is