asp-cp

@@ -34,7 +34,8 @@ then
   APPPATH=`type -path ascp`
   APPPATH=`dirname "$APPPATH"`
 fi
-if [ -z "$APPPATH" ]; then
+if [ -z "$APPPATH" ]
+then
   cat >&2 <<EOF
 
 Unable to find an Aspera ascp executable in any of the following locations:

asp-ls

@@ -10,8 +10,8 @@ my $dir    = shift;
 my $ftp    = new Net::FTP($server, Passive => 1)
   or die "Unable to connect to FTP server: $!";
 
-$ftp->login or die "Unable to log in to FTP server";
-$ftp->cwd($dir) or die "Unable to change to $dir";
+$ftp->login or die "Unable to log in to FTP server: ", $ftp->message;
+$ftp->cwd($dir) or die "Unable to change to $dir: ", $ftp->message;
 my $contents = $ftp->dir;
 die "Unable to list contents" unless defined $contents;
 

download-pubmed

+#!/bin/sh
+
+useasp=`has-asp`
+
+filter() {
+  while read fl
+  do
+    base=${fl%.xml.gz}
+    if [ -f "$fl" ]
+    then
+      continue
+    fi
+    if [ -f "$base.snt" ]
+    then
+      continue
+    fi
+    echo "$fl"
+  done
+}
+
+download() {
+  if [ "$useasp" == "true" ]
+  then
+    asp-ls "pubmed/$1" |
+    grep -v ".md5" | grep "xml.gz" |
+    filter |
+    asp-cp "pubmed/$1"
+  else
+    ftp-ls ftp.ncbi.nlm.nih.gov "pubmed/$1" |
+    grep -v ".md5" | grep "xml.gz" |
+    filter |
+    ftp-cp ftp.ncbi.nlm.nih.gov "pubmed/$1"
+  fi
+}
+
+if [ "$#" -eq 0 ]
+then
+  echo "Must indicate either baseline or updatefiles"
+  exit 1
+fi
+
+while [ "$#" -gt 0 ]
+do
+  sect="$1"
+  shift
+  download "$sect"
+  if [ $? -ne 0 ]
+  then
+    download "$sect"
+  fi
+done

download-sequence

+#!/bin/sh
+
+useasp=`has-asp`
+
+filt=""
+while [ "$#" -gt 0 ]
+do
+  filt=`echo "$filt -e $1"`
+  shift
+done
+
+download() {
+  if [ "$useasp" == "true" ]
+  then
+    asp-ls ncbi-asn1 |
+    grep "aso.gz" | eval "$filt" |
+    asp-cp ncbi-asn1
+  else
+    ftp-ls ftp.ncbi.nlm.nih.gov ncbi-asn1 |
+    grep "aso.gz" | eval "$filt" |
+    ftp-cp ftp.ncbi.nlm.nih.gov ncbi-asn1
+  fi
+}
+
+if [ -z "$filt" ]
+then
+  echo "Must have at least one sequence division abbreviation"
+  exit 1
+fi
+
+filt=`echo "grep$filt"`
+download
+if [ $? -ne 0 ]
+then
+  download
+fi

erase-pubmed

+#!/bin/sh
+
+target="$1"
+
+if [ "$#" -eq 0 ]
+then
+  echo "Must supply path to archive files"
+  exit 1
+fi
+
+osname=`uname -s | sed -e 's/_NT-.*$/_NT/; s/^MINGW[0-9]*/CYGWIN/'`
+if [ "$osname" == "CYGWIN_NT" -a -x /bin/cygpath ]
+then
+  target=`cygpath -w "$target"`
+fi
+
+target=${target%/}
+
+while read uid
+do
+  dir=`echo "$uid" | xtract -trie -gzip`
+  pth=`echo "$target/$dir"`
+  rm "$pth"
+done

fetch-pubmed

+#!/bin/sh
+
+target="$1"
+flag="none"
+
+if [ "$#" -eq 0 ]
+then
+  echo "Must supply path to archive files"
+  exit 1
+fi
+
+if [ "$#" -gt 1 ]
+then
+  flag="$1"
+  target="$2"
+fi
+
+osname=`uname -s | sed -e 's/_NT-.*$/_NT/; s/^MINGW[0-9]*/CYGWIN/'`
+if [ "$osname" == "CYGWIN_NT" -a -x /bin/cygpath ]
+then
+  target=`cygpath -w "$target"`
+fi
+
+xtract -archive "$target" -gzip -flag "$flag" \
+  -head "<PubmedArticleSet>" -tail "</PubmedArticleSet>"

ftp-cp

@@ -15,13 +15,17 @@ my @failed = ();
 sub fetch {
   my $fl = shift (@_);
   if (! -e $fl) {
-    $ftp->get($fl) or push (@failed, "$fl");
+    if (! $ftp->get($fl) ) {
+      my $msg = $ftp->message;
+      chomp $msg;
+      push (@failed, "$fl ($msg)");
+    }
   }
 }
 
-$ftp->login or die "Unable to log in to FTP server";
-$ftp->cwd($dir) or die "Unable to change to $dir";
-$ftp->binary or warn "Unable to set binary mode";
+$ftp->login or die "Unable to log in to FTP server: ", $ftp->message;
+$ftp->cwd($dir) or die "Unable to change to $dir: ", $ftp->message;
+$ftp->binary or warn "Unable to set binary mode: ", $ftp->message;
 
 if (@ARGV) {
 # file names on command line

ftp-ls

@@ -10,8 +10,8 @@ my $dir    = shift;
 my $ftp    = new Net::FTP($server, Passive => 1)
   or die "Unable to connect to FTP server: $!";
 
-$ftp->login or die "Unable to log in to FTP server";
-$ftp->cwd($dir) or die "Unable to change to $dir";
+$ftp->login or die "Unable to log in to FTP server: ", $ftp->message;
+$ftp->cwd($dir) or die "Unable to change to $dir: ", $ftp->message;
 my $contents = $ftp->dir;
 die "Unable to list contents" unless defined $contents;
 

has-asp

+#!/bin/sh
+DATAPATH="$1"
+APPPATH=
+# KEYPATH=
+KEYNAME=asperaweb_id_dsa.openssh
+
+case "`uname -s`" in
+  Darwin )
+    sysdir='/Applications/Aspera Connect.app/Contents/Resources'
+    sysdir2=/bin
+    userdir=$HOME$sysdir
+    ;;
+  CYGWIN_NT* )
+    sysdir='/cygdrive/c/Program Files/Aspera/Aspera Connect/bin'
+    sysdir2='/cygdrive/c/Program Files (x86)/Aspera/Aspera Connect/bin'
+    userdir="`cygpath -H`/$USER/AppData/Local/Programs/Aspera/Aspera Connect/bin"
+    ;;
+  * )
+    sysdir=/opt/aspera/bin
+    sysdir2=/bin
+    userdir=$HOME/.aspera/connect/bin
+    ;;
+esac
+for d in "$sysdir" "$sysdir2" "$userdir"
+do
+  if "$d/ascp" --version 2>&1 | grep '^Aspera' >/dev/null
+  then
+    APPPATH=$d
+    break
+  fi
+done
+if [ -z "$APPPATH" ]  &&  ascp --version 2>&1 | grep '^Aspera' >/dev/null
+then
+  APPPATH=`type -path ascp`
+  APPPATH=`dirname "$APPPATH"`
+fi
+if [ -z "$APPPATH" ]
+then
+  echo "false"
+  exit 1
+fi
+
+for d in "$APPPATH" "$sysdir" "$sysdir2" "$userdir"
+do
+  if [ -f "$d/../etc/$KEYNAME" ]
+  then
+    KEYPATH=$d/../etc
+    break
+  elif [ -f "$d/$KEYNAME" ]
+  then
+    KEYPATH=$d
+    break
+  fi
+done
+if [ -z "$KEYPATH" ]
+then
+  echo "false"
+  exit 1
+fi
+
+echo "true"

setup-deps.pl

@@ -5,9 +5,35 @@ use CPAN;
 use CPAN::HandleConfig;
 use CPAN::Shell;
 
+my $already_configured_cpan = 0;
 my $root;
 
 BEGIN {
+    sub CheckAvailability
+    {
+        my $code = "require $_[0]";
+        if (@_ > 1) {
+            $code .= "; $_[1]";
+        }
+        eval $code;
+        if ($@) {
+            if ($already_configured_cpan) {
+                print STDERR "Missing $_[0]; CPAN already initialized.\n";
+            } else {
+                print STDERR "Missing $_[0]; initializing CPAN.\n";
+                CPAN::HandleConfig->load(autoconfig => 1, auto_pick => 1,
+                                         doit => 1);
+                CPAN::Shell::setup_output;
+                CPAN::Index->reload;
+                $already_configured_cpan = 1;
+            }
+            return 0;
+        } else {
+            print STDERR "Found $_[0].\n";
+            return 1;
+        }
+    }
+
     alarm(3600);
 
     $root = $INC[0];
@@ -25,13 +51,11 @@ BEGIN {
         };
     }
     # $CPAN::DEBUG ||= $CPAN::DEBUG{'FTP'};
-    CPAN::HandleConfig->load(autoconfig => 1, auto_pick => 1, doit => 1);
-    CPAN::Shell::setup_output;
-    CPAN::Index->reload;
 
-    my $ll = CPAN::Shell->expandany('local::lib');
-    if ( ( ! $ll->inst_file  ||  $ll->inst_version =~ /^1\./)
+    if ( !CheckAvailability('local::lib',
+                            'die unless $local::lib::VERSION >= 2') 
         &&  ! -d "$root/aux/lib/perl5/local" ) {
+        my $ll = CPAN::Shell->expandany('local::lib');
         $ll->get;
         system('mkdir', '-p', "$root/aux/lib/perl5/local");
         system('cp', $ll->distribution->dir . "/lib/local/lib.pm",
@@ -47,10 +71,10 @@ my @lwp_deps = qw(Encode::Locale File::Listing
                   IO::Socket::SSL LWP::MediaTypes LWP::Protocol::https
                   Net::HTTP URI WWW::RobotRules Mozilla::CA);
 for my $module (@lwp_deps, 'Time::HiRes') {
-    if ( ! CPAN::Shell->expandany($module)->inst_file ) {
+    if ( ! CheckAvailability($module) ) {
         CPAN::Shell->install($module);
     }
 }
-if ( ! CPAN::Shell->expandany('LWP')->inst_file ) {
+if ( ! CheckAvailability('LWP') ) {
     CPAN::Shell->install('Bundle::LWP');
 }

setup.sh

@@ -13,7 +13,7 @@ cd "$DIR"
 
 mkdir -p _cpan/CPAN
 echo '1;' >> _cpan/CPAN/MyConfig.pm
-if ! perl -I_cpan setup-deps.pl </dev/null >setup-deps.log 2>&1
+if ! perl -I_cpan -Iaux/lib/perl5 setup-deps.pl </dev/null >setup-deps.log 2>&1
 then
   if grep '^read timeout.*HTTP' setup-deps.log >/dev/null
   then
@@ -31,7 +31,7 @@ then
   gzip -cd Mozilla-CA.tar.gz | tar xvf -
 fi
 
-osname=`uname -s | sed -e 's/[0-9.-]*$//'`
+osname=`uname -s | sed -e 's/_NT-.*$/_NT/; s/^MINGW[0-9]*/CYGWIN/'`
 cputype=`uname -m`
 case "$osname-$cputype" in
   Linux-x86_64 | Darwin-x86_64 | CYGWIN_NT-* )
@@ -43,11 +43,19 @@ if [ -f xtract."$osname" ]
 then
   chmod +x xtract."$osname"
 else
-  echo "Unable to download a prebuilt xtract executable; attempting to"
-  echo "build one from xtract.go.  A Perl fallback is also available, and"
-  echo "will be used if necessary, so please disregard any errors below."
+  if hash go 2>/dev/null
+  then
+    echo "Unable to download xtract executable; building from xtract.go."
     go build -o xtract."$osname" xtract.go
   fi
+  if [ ! -f xtract."$osname" ]
+  then
+    echo -e "Unable to download xtract executable. Please execute the following:\n\n"
+    echo -e "  ./ftp-cp ftp.ncbi.nlm.nih.gov /entrez/entrezdirect xtract.$osname.gz"
+    echo -e "  gunzip -f xtract.$osname.gz\n"
+    echo -e "  chmod +x xtract.$osname\n"
+  fi
+fi
 
 echo ""
 echo "Entrez Direct has been successfully downloaded and installed."

stash-pubmed

+#!/bin/sh
+
+target="$1"
+flag="none"
+
+if [ "$#" -eq 0 ]
+then
+  echo "Must supply path to archive files"
+  exit 1
+fi
+
+if [ "$#" -gt 1 ]
+then
+  flag="$1"
+  target="$2"
+fi
+
+osname=`uname -s | sed -e 's/_NT-.*$/_NT/; s/^MINGW[0-9]*/CYGWIN/'`
+if [ "$osname" == "CYGWIN_NT" -a -x /bin/cygpath ]
+then
+  target=`cygpath -w "$target"`
+fi
+
+# stashing on PC hangs if XML is sent through stdin, so decompress and use -input instead
+
+for fl in *.xml.gz
+do
+  base=${fl%.xml.gz}
+  if [ -f "$base.snt" ]
+  then
+    continue
+  fi
+  echo "$base"
+  gunzip -c "$fl" > "$base.xml"
+  xtract -input "$base.xml" -archive "$target" -gzip -flag "$flag" \
+    -index MedlineCitation/PMID -pattern PubmedArticle
+  touch "$base.snt"
+  rm "$base.xml"
+done

xtract

 #!/bin/sh
 PATH=/bin:/usr/bin
 export PATH
-compiled=$0.`uname -s | sed -e 's/[0-9.-]*$//'`
+osname=`uname -s | sed -e 's/_NT-.*$/_NT/; s/^MINGW[0-9]*/CYGWIN/'`
+compiled=$0."$osname"
 if [ -x "$compiled" ]
 then
   exec "$compiled" "$@"
 else
-    exec $0.pl -fallback "$@"
+  echo ""
+  echo "Unable to locate xtract executable. Please execute the following:"
+  echo ""
+  echo "  ftp-cp ftp.ncbi.nlm.nih.gov /entrez/entrezdirect xtract.$osname.gz"
+  echo "  gunzip -f xtract.$osname.gz"
+  echo "  chmod +x xtract.$osname"
+  echo ""
 fi