Loading Catrack.c +28 −44 Original line number Diff line number Diff line Loading @@ -66,44 +66,26 @@ int main(int argc, char *argv[]) // Open DB stub and get number of blocks { char *pwd, *root; int i, plen, index, isdam; FILE *dstub; char *dstub_name; int plen; DAZZ_STUB *stub; plen = strlen(argv[1]); if (strcmp(argv[1]+(plen-3),".dam") == 0) root = Root(argv[1],".dam"); else root = Root(argv[1],".db"); pwd = PathTo(argv[1]); prefix = Strdup(Catenate(pwd,PATHSEP,root,"."),"Allocating track name"); dstub = fopen(Catenate(pwd,"/",root,".db"),"r"); isdam = 0; if (dstub == NULL) { dstub = fopen(Catenate(pwd,"/",root,".dam"),"r"); isdam = 1; if (dstub == NULL) { fprintf(stderr,"%s: Cannot find %s either as a .db or a .dam\n",Prog_Name,root); exit (1); if (strcmp(argv[1]+(plen-3),".dam") == 0) { root = Root(argv[1],".dam"); stub = Read_DB_Stub(Catenate(pwd,"/",root,".dam"),0); } else { root = Root(argv[1],".db"); stub = Read_DB_Stub(Catenate(pwd,"/",root,".db"),0); } dstub_name = Strdup(Catenate(pwd,"/",root,isdam?".dam":".db"),"Allocating db file name"); if (dstub_name == NULL) exit (1); FSCANF(dstub,DB_NFILE,&nblocks) nblocks = stub->nblocks; for (i = 0; i < nblocks; i++) { char prolog[MAX_NAME], fname[MAX_NAME]; Free_DB_Stub(stub); FSCANF(dstub,DB_FDATA,&index,fname,prolog) } FSCANF(dstub,DB_NBLOCK,&nblocks) prefix = Strdup(Catenate(pwd,PATHSEP,root,"."),"Allocating track name"); fclose(dstub); free(dstub_name); free(pwd); free(root); } Loading Loading @@ -176,13 +158,15 @@ int main(int argc, char *argv[]) if (afile_name == NULL || dfile_name == NULL) goto error; afile = fopen(afile_name,"r"); if (afile == NULL) { free(afile_name); free(dfile_name); break; dfile = fopen(Numbered_Suffix(prefix,nfiles+1,Catenate(".",argv[c],".","data")),"r"); } dfile = fopen(dfile_name,"r"); if (dfile == NULL && errno != ENOENT) { fprintf(stderr,"%s: The file %s is corrupted\n",Prog_Name,dfile_name); { fprintf(stderr,"%s: Cannot find/open data file %s\n",Prog_Name,dfile_name); goto error; } Loading Loading @@ -325,6 +309,8 @@ int main(int argc, char *argv[]) nfiles += 1; if (dfile != NULL) fclose(dfile); free(dfile_name); free(afile_name); } if (nfiles == 0) Loading Loading @@ -363,10 +349,8 @@ int main(int argc, char *argv[]) } if (nfiles != nblocks) { fprintf(stderr,"%s: Did not catenate all tracks of DB (nfiles %d != nblocks %d)\n", fprintf(stderr,"%s: Warning: Did not catenate all tracks of DB (nfiles %d != nblocks %d)\n", Prog_Name,nfiles,nblocks); goto error; } FCLOSE(aout); if (dout != NULL) Loading @@ -376,7 +360,7 @@ int main(int argc, char *argv[]) { int i; char *name; for (i = 1; i <= nblocks ;i++) for (i = 1; i <= nfiles ;i++) { name = Numbered_Suffix(prefix,i,Catenate(".",argv[c],".","anno")); if (unlink(name) != 0) fprintf(stderr,"%s: [WARNING] Couldn't delete file %s\n",Prog_Name,name); Loading DB.c +211 −6 Original line number Diff line number Diff line Loading @@ -465,6 +465,205 @@ void Change_Read(char *s) } /******************************************************************************************* * * DB STUB HANDLING ROUTINES * ********************************************************************************************/ // Read the contents of the DB stub file at "path" and return it encoded in a DAZZ_STUB // structure. This is allocated by the routine. "path" is assumed to be the complete // name of the file. DAZZ_STUB *Read_DB_Stub(char *path, int what) { FILE *dbfile; DAZZ_STUB *stub; char buf1[MAX_NAME+100]; char buf2[MAX_NAME+100]; int nread; int i; int nfiles; int nblocks; int64 size; int all, cutoff; dbfile = Fopen(path,"r"); if (dbfile == NULL) EXIT(NULL); stub = Malloc(sizeof(DAZZ_STUB),"Allocating DB stub record"); if (stub == NULL) EXIT(NULL); stub->nreads = NULL; stub->fname = NULL; stub->prolog = NULL; stub->ublocks = NULL; stub->tblocks = NULL; if (fscanf(dbfile,DB_NFILE,&nfiles) != 1) goto stub_trash; if (what & DB_STUB_NREADS) { stub->nreads = (int *) Malloc(sizeof(int)*(nfiles+1),"Allocating DB stub record"); if (stub->nreads == NULL) goto stub_error; stub->nreads += 1; } if (what & DB_STUB_FILES) { stub->fname = (char **) Malloc(sizeof(char *)*(nfiles+1),"Allocating DB stub record"); if (stub->fname == NULL) goto stub_error; stub->fname += 1; stub->nfiles = nfiles; for (i = 0; i < nfiles; i++) stub->fname[i] = NULL; } if (what & DB_STUB_PROLOGS) { stub->prolog = (char **) Malloc(sizeof(char *)*(nfiles+1),"Allocating DB stub record"); if (stub->prolog == NULL) goto stub_error; stub->prolog += 1; for (i = 0; i < nfiles; i++) stub->prolog[i] = NULL; } for (i = 0; i < nfiles; i++) { if (fscanf(dbfile,DB_FDATA,&nread,buf1,buf2) != 3) goto stub_trash; if (what & DB_STUB_NREADS) stub->nreads[i] = nread; if (what & DB_STUB_FILES) { stub->fname[i] = Strdup(buf1,"Alloacting DB stub record"); if (stub->fname[i] == NULL) goto stub_error; } if (what & DB_STUB_PROLOGS) { stub->prolog[i] = Strdup(buf2,"Alloacting DB stub record"); if (stub->prolog[i] == NULL) goto stub_error; } } if (fscanf(dbfile,DB_NBLOCK,&nblocks) != 1) goto stub_trash; if (fscanf(dbfile,DB_PARAMS,&size,&cutoff,&all) != 3) goto stub_trash; if (what & DB_STUB_BLOCKS) { stub->ublocks = (int *) Malloc(sizeof(int)*(nblocks+1),"Allocating DB stub record"); stub->tblocks = (int *) Malloc(sizeof(int)*(nblocks+1),"Allocating DB stub record"); if (stub->ublocks == NULL || stub->tblocks == NULL) goto stub_error; for (i = 0; i <= nblocks; i++) if (fscanf(dbfile,DB_BDATA,stub->ublocks+i,stub->tblocks+i) != 2) goto stub_trash; } fclose(dbfile); stub->nfiles = nfiles; stub->all = all; stub->cutoff = cutoff; stub->bsize = size; stub->nblocks = nblocks; return (stub); stub_trash: EPRINTF(EPLACE,"%s: Stub file %s is junk\n",Prog_Name,path); stub_error: Free_DB_Stub(stub); EXIT(NULL); } // Read the DB stub file "path" and extract the read index range [*first,*last) // for block n, for the trimmed DB if trim is set, the untrimmed DB otherwise. int Fetch_Block_Range(char *path, int trim, int n, int *first, int *last) { FILE *dbfile; char buffer[2*MAX_NAME+100]; int nfiles; int nblocks; int64 size; int all, cutoff; int tfirst, tlast; int ufirst, ulast; int i; dbfile = Fopen(path,"r"); if (dbfile == NULL) EXIT(1); if (fscanf(dbfile,DB_NFILE,&nfiles) != 1) goto stub_error; for (i = 0; i < nfiles; i++) if (fgets(buffer,2*MAX_NAME+100,dbfile) == NULL) goto stub_error; if (fscanf(dbfile,DB_NBLOCK,&nblocks) != 1) goto stub_error; if (n < 0 || n >= nblocks) { *first = *last = -1; return (0); } if (fscanf(dbfile,DB_PARAMS,&size,&cutoff,&all) != 3) goto stub_error; for (i = 1; i <= n; i++) if (fscanf(dbfile,DB_BDATA,&ufirst,&tfirst) != 2) goto stub_error; if (fscanf(dbfile,DB_BDATA,&ulast,&tlast) != 2) goto stub_error; fclose(dbfile); if (trim) { *first = tfirst; *last = tlast; } else { *first = ufirst; *last = ulast; } return (0); stub_error: EPRINTF(EPLACE,"%s: Stub file %s is junk\n",Prog_Name,path); EXIT(1); } // Free a DAZZ_STUB data structure returned by Read_DB_Stub void Free_DB_Stub(DAZZ_STUB *stub) { int i; if (stub == NULL) return; if (stub->fname != NULL) { for (i = 0; i < stub->nfiles; i++) free(stub->fname[i]); free(stub->fname-1); } if (stub->prolog != NULL) { for (i = 0; i < stub->nfiles; i++) free(stub->prolog[i]); free(stub->prolog-1); } if (stub->nreads != NULL) free(stub->nreads-1); free(stub->ublocks); free(stub->tblocks); free(stub); } /******************************************************************************************* * * DB OPEN, TRIM, SIZE_OF, LIST_FILES & CLOSE ROUTINES Loading Loading @@ -770,6 +969,7 @@ void Trim_DB(DAZZ_DB *db) { memmove(anno+j,anno+r,size); j += size; } record->anno = Realloc(record->anno,record->size*j,NULL); } else if (size == 4) { int *anno4 = (int *) (record->anno); Loading @@ -783,6 +983,7 @@ void Trim_DB(DAZZ_DB *db) j += 1; } record->alen = Realloc(record->alen,sizeof(int)*j,NULL); record->anno = Realloc(record->anno,record->size*(j+1),NULL); } else // size == 8 { int64 *anno8 = (int64 *) (record->anno); Loading @@ -796,8 +997,8 @@ void Trim_DB(DAZZ_DB *db) j += 1; } record->alen = Realloc(record->alen,sizeof(int)*j,NULL); } record->anno = Realloc(record->anno,record->size*(j+1),NULL); } record->nreads = j; } Loading @@ -805,7 +1006,7 @@ void Trim_DB(DAZZ_DB *db) totlen = maxlen = 0; for (j = i = 0; i < nreads; i++) { f = reads[i].flags; if ((f & DB_CSS) == 0) if ((f & DB_CCS) == 0) css = 0; r = reads[i].rlen; if ((f & DB_BEST) >= allflag && r >= cutoff) Loading @@ -814,9 +1015,9 @@ void Trim_DB(DAZZ_DB *db) maxlen = r; reads[j] = reads[i]; if (css) reads[j++].flags |= DB_CSS; reads[j++].flags |= DB_CCS; else reads[j++].flags &= ~DB_CSS; reads[j++].flags &= ~DB_CCS; css = 1; } } Loading Loading @@ -1538,7 +1739,7 @@ static int Late_Track_Trim(DAZZ_DB *db, DAZZ_TRACK *track, int ispart) } r += size; } memmove(anno+j,anno+r,size); track->anno = Realloc(track->anno,track->size*j,NULL); } else if (size == 4) { int *anno4 = (int *) (track->anno); Loading @@ -1558,6 +1759,8 @@ static int Late_Track_Trim(DAZZ_DB *db, DAZZ_TRACK *track, int ispart) } } track->data = Realloc(track->data,anno4[j],NULL); track->alen = Realloc(track->alen,sizeof(int)*j,NULL); track->anno = Realloc(track->anno,track->size*(j+1),NULL); } else // size == 8 { int64 *anno8 = (int64 *) (track->anno); Loading @@ -1577,9 +1780,10 @@ static int Late_Track_Trim(DAZZ_DB *db, DAZZ_TRACK *track, int ispart) } } track->data = Realloc(track->data,anno8[j],NULL); } track->alen = Realloc(track->alen,sizeof(int)*j,NULL); track->anno = Realloc(track->anno,track->size*(j+1),NULL); } } fclose(indx); return (0); Loading Loading @@ -2076,6 +2280,7 @@ void Close_Track(DAZZ_DB *db, DAZZ_TRACK *track) for (record = db->tracks; record != NULL; record = record->next) { if (track == record) { free(record->anno); free(record->alen); if (record->loaded) free(record->data); else Loading DB.h +48 −6 Original line number Diff line number Diff line Loading @@ -249,7 +249,7 @@ char *Numbered_Suffix(char *left, int num, char *right); // DB-related utilities void Print_Number(int64 num, int width, FILE *out); // Print readable big integer void Print_Number(int64 num, int width, FILE *out); // Print big integer with commas int Number_Digits(int64 num); // Return # of digits in printed number #define COMPRESSED_LEN(len) (((len)+3) >> 2) Loading @@ -274,7 +274,7 @@ void Number_Arrow(char *s); // Convert arrow pw string from letters to number ********************************************************************************************/ #define DB_QV 0x03ff // Mask for 3-digit quality value #define DB_CSS 0x0400 // This is the second or later of a group of subreads from a given insert #define DB_CCS 0x0400 // This is the second or later of a group of subreads from a given insert #define DB_BEST 0x0800 // This is the "best" subread of a given insert (may be the only 1) #define DB_ARROW 0x2 // DB is an arrow DB Loading @@ -300,7 +300,7 @@ typedef struct // contains the variable length data // data != NULL && size == 8: anno is an array of nreads+1 int64's and data[anno[i]..anno[i+1]) // contains the variable length data // if open is set then the data is not loaded if present, rather data is an open file pointer // if loaded is set then the data is not loaded if present, rather data is an open file pointer // set for reading. typedef struct _track Loading Loading @@ -365,6 +365,24 @@ typedef struct int loaded; // Are arrow vectors loaded in memory? } DAZZ_ARROW; // Every DB is referred to by an ASCII stub file with extension .db or .dam. This file // contains the information about the SMRT cells in the DB and the current division of // the DB into blocks for HPC processing. This file can be read into the following // data structure: typedef struct { int nfiles; // Number of files/SMRT cells in DB int *nreads; // [0..nfiles) = # of reads from cell char **fname; // [0..nfiles) = file name of cell char **prolog; // [0..nfiles) = fasta header prolog for cell int all; // Keep only best read from each well? int cutoff; // Trim reads less than cutoff int64 bsize; // Target size for blocks int nblocks; // Number of blocks for DB int *ublocks; // [0..nblcoks] = index of 1st read in block in untrimmed DB int *tblocks; // [0..nblcoks] = index of 1st read in block in trimmed DB } DAZZ_STUB; // The DB record holds all information about the current state of an active DB including an // array of DAZZ_READS, one per read, and a linked list of DAZZ_TRACKs the first of which // is always a DAZZ_QV pseudo-track (if the QVs have been loaded). Loading Loading @@ -416,6 +434,30 @@ typedef struct #define DB_PARAMS "size = %11lld cutoff = %9d all = %1d\n" // block size, len cutoff, all in well #define DB_BDATA " %9d %9d\n" // First read index (untrimmed), first read index (trimmed) // Read the specified contents of the DB stub file at "path" and return it encoded in a DAZZ_STUB // structure. This is allocated by the routine. "path" is assumed to be the complete // name of the file. If all flags are off, then just the scalar parts of the stub // are returned (i.e. nfiles, all, cutoff, bsize, nblocks). Returns NULL if an error // occured in INTERACTIVE mode #define DB_STUB_NREADS 0x1 #define DB_STUB_FILES 0x2 #define DB_STUB_PROLOGS 0x4 #define DB_STUB_BLOCKS 0x8 DAZZ_STUB *Read_DB_Stub(char *path, int what); // Read the DB stub file "path" and extract the read index range [*first,*last) // for block n, for the trimmed DB if trim is set, the untrimmed DB otherwise. // If n is out of range first and last will be set to -1. Returns 0 unless // an error occurs in INTERACTIVE mode in which case it returns 1. int Fetch_Block_Range(char *path, int trim, int n, int *first, int *last); // Free a DAZZ_STUB data structure returned by Read_DB_Stub void Free_DB_Stub(DAZZ_STUB *stub); /******************************************************************************************* * Loading @@ -428,13 +470,13 @@ typedef struct // (not containing a . !). // A DAM is basically a DB except that: // 1. there are no QV's, instead .coff points the '\0' terminated fasta header of the read // in the file .<dam>.hdr file // 1. there are no QV's, instead .coff points to the '\0' terminated fasta header of the read // in an additional file: .DB.hdr // 2. .origin contains the contig # of the read within a fasta entry (assembly sequences // contain N-separated contigs), and .fpulse the first base of the contig in the // fasta entry // Open the given database or dam, "path" into the supplied DAZZ_DB record "db". If the name has // Open the given database or dam, "path", into the supplied DAZZ_DB record "db". If the name has // a part # in it then just the part is opened. The index array is allocated (for all or // just the part) and read in. // Return status of routine: Loading DBdump.c +18 −47 Original line number Diff line number Diff line Loading @@ -85,6 +85,7 @@ static int qv_map[51] = int main(int argc, char *argv[]) { DAZZ_DB _db, *db = &_db; DAZZ_STUB *stub; int Quiva_DB, Arrow_DB; int FirstRead; FILE *hdrs = NULL; Loading Loading @@ -291,8 +292,6 @@ int main(int argc, char *argv[]) // If get prolog and file names and index ranges from the .db or .dam file { char *pwd, *root; FILE *dstub; char *dstub_name; int i; if (DAM) Loading @@ -303,41 +302,22 @@ int main(int argc, char *argv[]) if (db->part > 0) *rindex(root,'.') = '\0'; if (DAM) dstub_name = Strdup(Catenate(pwd,"/",root,".dam"),"Allocating dam file name"); stub = Read_DB_Stub(Catenate(pwd,"/",root,".dam"), DB_STUB_NREADS|DB_STUB_FILES|DB_STUB_PROLOGS); else dstub_name = Strdup(Catenate(pwd,"/",root,".db"),"Allocating db file name"); dstub = Fopen(dstub_name,"r"); if (dstub_name == NULL || dstub == NULL) exit (1); stub = Read_DB_Stub(Catenate(pwd,"/",root,".db"), DB_STUB_NREADS|DB_STUB_FILES|DB_STUB_PROLOGS); free(pwd); free(root); FSCANF(dstub,DB_NFILE,&nfiles) fhead = (char **) Malloc(sizeof(char *)*nfiles,"Allocating file list"); ffile = (char **) Malloc(sizeof(char *)*(nfiles+1),"Allocating file list"); findx = (int *) Malloc(sizeof(int *)*(nfiles+1),"Allocating file index"); if (fhead == NULL || ffile == NULL || findx == NULL) exit (1); fhead = stub->prolog; ffile = stub->fname; findx = stub->nreads; nfiles = stub->nfiles; findx += 1; findx[-1] = 0; ffile += 1; ffile[-1] = empty; for (i = 0; i < nfiles; i++) { char prolog[MAX_NAME], fname[MAX_NAME]; FSCANF(dstub,DB_FDATA,findx+i,fname,prolog) if ((fhead[i] = Strdup(prolog,"Adding to file list")) == NULL) exit (1); if ((ffile[i] = Strdup(fname,"Adding to file list")) == NULL) exit (1); } free(dstub_name); fclose(dstub); // If TRIM (the default) then "trim" prolog ranges and the DB if (TRIM) Loading Loading @@ -547,11 +527,11 @@ int main(int argc, char *argv[]) c += 2; } if (map > 0 && findx[map-1] <= b+FirstRead && b+FirstRead < findx[map]) if (map > 0 && findx[map-1] <= b && b < findx[map]) ; else { map = 0; while (b + FirstRead >= findx[map]) while (b >= findx[map]) map += 1; map -= 1; } Loading @@ -566,7 +546,7 @@ int main(int argc, char *argv[]) noreads += 1; if (DOFLN && i+FirstRead >= findx[map]) if (DOFLN && i >= findx[map]) { int ten; if (strcmp(ffile[map+1],ffile[last]) != 0) Loading Loading @@ -595,7 +575,7 @@ int main(int argc, char *argv[]) hdrmax = ten; hdrtot += ten; } else if (i+FirstRead >= findx[map]) else if (i >= findx[map]) { map += 1; ten = strlen(fhead[map]); Loading Loading @@ -713,11 +693,11 @@ int main(int argc, char *argv[]) c += 2; } if (map > 0 && findx[map-1] <= b+FirstRead && b+FirstRead < findx[map]) if (map > 0 && findx[map-1] <= b && b < findx[map]) ; else { map = 0; while (b + FirstRead >= findx[map]) while (b >= findx[map]) map += 1; map -= 1; } Loading @@ -736,7 +716,7 @@ int main(int argc, char *argv[]) flags = r->flags; qv = (flags & DB_QV); if (DOFLN && i+FirstRead >= findx[map]) if (DOFLN && i >= findx[map]) { if (strcmp(ffile[map+1],ffile[last]) != 0) { PRINTF("F %ld %s\n",strlen(ffile[map+1]),ffile[map+1]) last = map+1; Loading @@ -756,7 +736,7 @@ int main(int argc, char *argv[]) PRINTF("L %d %d %d\n",r->origin,r->fpulse,r->fpulse+len) } else { if (i+FirstRead >= findx[map]) { if (i >= findx[map]) { map += 1; PRINTF("H %ld %s\n",strlen(fhead[map]),fhead[map]) } Loading Loading @@ -854,16 +834,7 @@ int main(int argc, char *argv[]) if (DAM) fclose(hdrs); else { int i; for (i = 0; i < nfiles; i++) { free(fhead[i]); free(ffile[i]); } free(fhead); free(ffile-1); free(findx-1); } Free_DB_Stub(stub); Close_DB(db); exit (0); Loading DBshow.c +14 −41 Original line number Diff line number Diff line Loading @@ -81,12 +81,13 @@ int next_read(File_Iterator *it) int main(int argc, char *argv[]) { DAZZ_DB _db, *db = &_db; DAZZ_STUB *stub; FILE *hdrs = NULL; char *hdrs_name = NULL; int nfiles; char **flist = NULL; int *findx = NULL; char **flist; int *findx; int reps, *pts; int input_pts; Loading Loading @@ -264,44 +265,21 @@ int main(int argc, char *argv[]) if (!DAM) { char *pwd, *root; FILE *dstub; char *dstub_name; int i; root = Root(argv[1],".db"); pwd = PathTo(argv[1]); if (db->part > 0) *rindex(root,'.') = '\0'; dstub_name = Strdup(Catenate(pwd,"/",root,".db"),"Allocating db file name"); dstub = Fopen(dstub_name,"r"); if (dstub_name == NULL || dstub == NULL) exit (1); free(pwd); free(root); FSCANF(dstub,DB_NFILE,&nfiles) flist = (char **) Malloc(sizeof(char *)*nfiles,"Allocating file list"); findx = (int *) Malloc(sizeof(int *)*(nfiles+1),"Allocating file index"); if (flist == NULL || findx == NULL) exit (1); stub = Read_DB_Stub(Catenate(pwd,"/",root,".db"),DB_STUB_NREADS|DB_STUB_PROLOGS); findx += 1; findx[-1] = 0; for (i = 0; i < nfiles; i++) { char prolog[MAX_NAME], fname[MAX_NAME]; FSCANF(dstub,DB_FDATA,findx+i,fname,prolog) if ((flist[i] = Strdup(prolog,"Adding to file list")) == NULL) exit (1); } fclose(dstub); free(dstub_name); nfiles = stub->nfiles; flist = stub->prolog; findx = stub->nreads; // If TRIM (the default) then "trim" prolog ranges and the DB findx[-1] = 0; if (TRIM) { int nid, oid, lid; int cutoff, allflag; Loading @@ -327,8 +305,9 @@ int main(int argc, char *argv[]) } } else if (db->part > 0) { for (i = 0; i < nfiles; i++) else { if (db->part > 0) for (i = 0; i < nfiles; i++) findx[i] -= db->ufirst; } } Loading Loading @@ -660,13 +639,7 @@ int main(int argc, char *argv[]) if (DAM) fclose(hdrs); else { int i; for (i = 0; i < nfiles; i++) free(flist[i]); free(flist); free(findx-1); } Free_DB_Stub(stub); Close_DB(db); exit (0); Loading Loading
Catrack.c +28 −44 Original line number Diff line number Diff line Loading @@ -66,44 +66,26 @@ int main(int argc, char *argv[]) // Open DB stub and get number of blocks { char *pwd, *root; int i, plen, index, isdam; FILE *dstub; char *dstub_name; int plen; DAZZ_STUB *stub; plen = strlen(argv[1]); if (strcmp(argv[1]+(plen-3),".dam") == 0) root = Root(argv[1],".dam"); else root = Root(argv[1],".db"); pwd = PathTo(argv[1]); prefix = Strdup(Catenate(pwd,PATHSEP,root,"."),"Allocating track name"); dstub = fopen(Catenate(pwd,"/",root,".db"),"r"); isdam = 0; if (dstub == NULL) { dstub = fopen(Catenate(pwd,"/",root,".dam"),"r"); isdam = 1; if (dstub == NULL) { fprintf(stderr,"%s: Cannot find %s either as a .db or a .dam\n",Prog_Name,root); exit (1); if (strcmp(argv[1]+(plen-3),".dam") == 0) { root = Root(argv[1],".dam"); stub = Read_DB_Stub(Catenate(pwd,"/",root,".dam"),0); } else { root = Root(argv[1],".db"); stub = Read_DB_Stub(Catenate(pwd,"/",root,".db"),0); } dstub_name = Strdup(Catenate(pwd,"/",root,isdam?".dam":".db"),"Allocating db file name"); if (dstub_name == NULL) exit (1); FSCANF(dstub,DB_NFILE,&nblocks) nblocks = stub->nblocks; for (i = 0; i < nblocks; i++) { char prolog[MAX_NAME], fname[MAX_NAME]; Free_DB_Stub(stub); FSCANF(dstub,DB_FDATA,&index,fname,prolog) } FSCANF(dstub,DB_NBLOCK,&nblocks) prefix = Strdup(Catenate(pwd,PATHSEP,root,"."),"Allocating track name"); fclose(dstub); free(dstub_name); free(pwd); free(root); } Loading Loading @@ -176,13 +158,15 @@ int main(int argc, char *argv[]) if (afile_name == NULL || dfile_name == NULL) goto error; afile = fopen(afile_name,"r"); if (afile == NULL) { free(afile_name); free(dfile_name); break; dfile = fopen(Numbered_Suffix(prefix,nfiles+1,Catenate(".",argv[c],".","data")),"r"); } dfile = fopen(dfile_name,"r"); if (dfile == NULL && errno != ENOENT) { fprintf(stderr,"%s: The file %s is corrupted\n",Prog_Name,dfile_name); { fprintf(stderr,"%s: Cannot find/open data file %s\n",Prog_Name,dfile_name); goto error; } Loading Loading @@ -325,6 +309,8 @@ int main(int argc, char *argv[]) nfiles += 1; if (dfile != NULL) fclose(dfile); free(dfile_name); free(afile_name); } if (nfiles == 0) Loading Loading @@ -363,10 +349,8 @@ int main(int argc, char *argv[]) } if (nfiles != nblocks) { fprintf(stderr,"%s: Did not catenate all tracks of DB (nfiles %d != nblocks %d)\n", fprintf(stderr,"%s: Warning: Did not catenate all tracks of DB (nfiles %d != nblocks %d)\n", Prog_Name,nfiles,nblocks); goto error; } FCLOSE(aout); if (dout != NULL) Loading @@ -376,7 +360,7 @@ int main(int argc, char *argv[]) { int i; char *name; for (i = 1; i <= nblocks ;i++) for (i = 1; i <= nfiles ;i++) { name = Numbered_Suffix(prefix,i,Catenate(".",argv[c],".","anno")); if (unlink(name) != 0) fprintf(stderr,"%s: [WARNING] Couldn't delete file %s\n",Prog_Name,name); Loading
DB.c +211 −6 Original line number Diff line number Diff line Loading @@ -465,6 +465,205 @@ void Change_Read(char *s) } /******************************************************************************************* * * DB STUB HANDLING ROUTINES * ********************************************************************************************/ // Read the contents of the DB stub file at "path" and return it encoded in a DAZZ_STUB // structure. This is allocated by the routine. "path" is assumed to be the complete // name of the file. DAZZ_STUB *Read_DB_Stub(char *path, int what) { FILE *dbfile; DAZZ_STUB *stub; char buf1[MAX_NAME+100]; char buf2[MAX_NAME+100]; int nread; int i; int nfiles; int nblocks; int64 size; int all, cutoff; dbfile = Fopen(path,"r"); if (dbfile == NULL) EXIT(NULL); stub = Malloc(sizeof(DAZZ_STUB),"Allocating DB stub record"); if (stub == NULL) EXIT(NULL); stub->nreads = NULL; stub->fname = NULL; stub->prolog = NULL; stub->ublocks = NULL; stub->tblocks = NULL; if (fscanf(dbfile,DB_NFILE,&nfiles) != 1) goto stub_trash; if (what & DB_STUB_NREADS) { stub->nreads = (int *) Malloc(sizeof(int)*(nfiles+1),"Allocating DB stub record"); if (stub->nreads == NULL) goto stub_error; stub->nreads += 1; } if (what & DB_STUB_FILES) { stub->fname = (char **) Malloc(sizeof(char *)*(nfiles+1),"Allocating DB stub record"); if (stub->fname == NULL) goto stub_error; stub->fname += 1; stub->nfiles = nfiles; for (i = 0; i < nfiles; i++) stub->fname[i] = NULL; } if (what & DB_STUB_PROLOGS) { stub->prolog = (char **) Malloc(sizeof(char *)*(nfiles+1),"Allocating DB stub record"); if (stub->prolog == NULL) goto stub_error; stub->prolog += 1; for (i = 0; i < nfiles; i++) stub->prolog[i] = NULL; } for (i = 0; i < nfiles; i++) { if (fscanf(dbfile,DB_FDATA,&nread,buf1,buf2) != 3) goto stub_trash; if (what & DB_STUB_NREADS) stub->nreads[i] = nread; if (what & DB_STUB_FILES) { stub->fname[i] = Strdup(buf1,"Alloacting DB stub record"); if (stub->fname[i] == NULL) goto stub_error; } if (what & DB_STUB_PROLOGS) { stub->prolog[i] = Strdup(buf2,"Alloacting DB stub record"); if (stub->prolog[i] == NULL) goto stub_error; } } if (fscanf(dbfile,DB_NBLOCK,&nblocks) != 1) goto stub_trash; if (fscanf(dbfile,DB_PARAMS,&size,&cutoff,&all) != 3) goto stub_trash; if (what & DB_STUB_BLOCKS) { stub->ublocks = (int *) Malloc(sizeof(int)*(nblocks+1),"Allocating DB stub record"); stub->tblocks = (int *) Malloc(sizeof(int)*(nblocks+1),"Allocating DB stub record"); if (stub->ublocks == NULL || stub->tblocks == NULL) goto stub_error; for (i = 0; i <= nblocks; i++) if (fscanf(dbfile,DB_BDATA,stub->ublocks+i,stub->tblocks+i) != 2) goto stub_trash; } fclose(dbfile); stub->nfiles = nfiles; stub->all = all; stub->cutoff = cutoff; stub->bsize = size; stub->nblocks = nblocks; return (stub); stub_trash: EPRINTF(EPLACE,"%s: Stub file %s is junk\n",Prog_Name,path); stub_error: Free_DB_Stub(stub); EXIT(NULL); } // Read the DB stub file "path" and extract the read index range [*first,*last) // for block n, for the trimmed DB if trim is set, the untrimmed DB otherwise. int Fetch_Block_Range(char *path, int trim, int n, int *first, int *last) { FILE *dbfile; char buffer[2*MAX_NAME+100]; int nfiles; int nblocks; int64 size; int all, cutoff; int tfirst, tlast; int ufirst, ulast; int i; dbfile = Fopen(path,"r"); if (dbfile == NULL) EXIT(1); if (fscanf(dbfile,DB_NFILE,&nfiles) != 1) goto stub_error; for (i = 0; i < nfiles; i++) if (fgets(buffer,2*MAX_NAME+100,dbfile) == NULL) goto stub_error; if (fscanf(dbfile,DB_NBLOCK,&nblocks) != 1) goto stub_error; if (n < 0 || n >= nblocks) { *first = *last = -1; return (0); } if (fscanf(dbfile,DB_PARAMS,&size,&cutoff,&all) != 3) goto stub_error; for (i = 1; i <= n; i++) if (fscanf(dbfile,DB_BDATA,&ufirst,&tfirst) != 2) goto stub_error; if (fscanf(dbfile,DB_BDATA,&ulast,&tlast) != 2) goto stub_error; fclose(dbfile); if (trim) { *first = tfirst; *last = tlast; } else { *first = ufirst; *last = ulast; } return (0); stub_error: EPRINTF(EPLACE,"%s: Stub file %s is junk\n",Prog_Name,path); EXIT(1); } // Free a DAZZ_STUB data structure returned by Read_DB_Stub void Free_DB_Stub(DAZZ_STUB *stub) { int i; if (stub == NULL) return; if (stub->fname != NULL) { for (i = 0; i < stub->nfiles; i++) free(stub->fname[i]); free(stub->fname-1); } if (stub->prolog != NULL) { for (i = 0; i < stub->nfiles; i++) free(stub->prolog[i]); free(stub->prolog-1); } if (stub->nreads != NULL) free(stub->nreads-1); free(stub->ublocks); free(stub->tblocks); free(stub); } /******************************************************************************************* * * DB OPEN, TRIM, SIZE_OF, LIST_FILES & CLOSE ROUTINES Loading Loading @@ -770,6 +969,7 @@ void Trim_DB(DAZZ_DB *db) { memmove(anno+j,anno+r,size); j += size; } record->anno = Realloc(record->anno,record->size*j,NULL); } else if (size == 4) { int *anno4 = (int *) (record->anno); Loading @@ -783,6 +983,7 @@ void Trim_DB(DAZZ_DB *db) j += 1; } record->alen = Realloc(record->alen,sizeof(int)*j,NULL); record->anno = Realloc(record->anno,record->size*(j+1),NULL); } else // size == 8 { int64 *anno8 = (int64 *) (record->anno); Loading @@ -796,8 +997,8 @@ void Trim_DB(DAZZ_DB *db) j += 1; } record->alen = Realloc(record->alen,sizeof(int)*j,NULL); } record->anno = Realloc(record->anno,record->size*(j+1),NULL); } record->nreads = j; } Loading @@ -805,7 +1006,7 @@ void Trim_DB(DAZZ_DB *db) totlen = maxlen = 0; for (j = i = 0; i < nreads; i++) { f = reads[i].flags; if ((f & DB_CSS) == 0) if ((f & DB_CCS) == 0) css = 0; r = reads[i].rlen; if ((f & DB_BEST) >= allflag && r >= cutoff) Loading @@ -814,9 +1015,9 @@ void Trim_DB(DAZZ_DB *db) maxlen = r; reads[j] = reads[i]; if (css) reads[j++].flags |= DB_CSS; reads[j++].flags |= DB_CCS; else reads[j++].flags &= ~DB_CSS; reads[j++].flags &= ~DB_CCS; css = 1; } } Loading Loading @@ -1538,7 +1739,7 @@ static int Late_Track_Trim(DAZZ_DB *db, DAZZ_TRACK *track, int ispart) } r += size; } memmove(anno+j,anno+r,size); track->anno = Realloc(track->anno,track->size*j,NULL); } else if (size == 4) { int *anno4 = (int *) (track->anno); Loading @@ -1558,6 +1759,8 @@ static int Late_Track_Trim(DAZZ_DB *db, DAZZ_TRACK *track, int ispart) } } track->data = Realloc(track->data,anno4[j],NULL); track->alen = Realloc(track->alen,sizeof(int)*j,NULL); track->anno = Realloc(track->anno,track->size*(j+1),NULL); } else // size == 8 { int64 *anno8 = (int64 *) (track->anno); Loading @@ -1577,9 +1780,10 @@ static int Late_Track_Trim(DAZZ_DB *db, DAZZ_TRACK *track, int ispart) } } track->data = Realloc(track->data,anno8[j],NULL); } track->alen = Realloc(track->alen,sizeof(int)*j,NULL); track->anno = Realloc(track->anno,track->size*(j+1),NULL); } } fclose(indx); return (0); Loading Loading @@ -2076,6 +2280,7 @@ void Close_Track(DAZZ_DB *db, DAZZ_TRACK *track) for (record = db->tracks; record != NULL; record = record->next) { if (track == record) { free(record->anno); free(record->alen); if (record->loaded) free(record->data); else Loading
DB.h +48 −6 Original line number Diff line number Diff line Loading @@ -249,7 +249,7 @@ char *Numbered_Suffix(char *left, int num, char *right); // DB-related utilities void Print_Number(int64 num, int width, FILE *out); // Print readable big integer void Print_Number(int64 num, int width, FILE *out); // Print big integer with commas int Number_Digits(int64 num); // Return # of digits in printed number #define COMPRESSED_LEN(len) (((len)+3) >> 2) Loading @@ -274,7 +274,7 @@ void Number_Arrow(char *s); // Convert arrow pw string from letters to number ********************************************************************************************/ #define DB_QV 0x03ff // Mask for 3-digit quality value #define DB_CSS 0x0400 // This is the second or later of a group of subreads from a given insert #define DB_CCS 0x0400 // This is the second or later of a group of subreads from a given insert #define DB_BEST 0x0800 // This is the "best" subread of a given insert (may be the only 1) #define DB_ARROW 0x2 // DB is an arrow DB Loading @@ -300,7 +300,7 @@ typedef struct // contains the variable length data // data != NULL && size == 8: anno is an array of nreads+1 int64's and data[anno[i]..anno[i+1]) // contains the variable length data // if open is set then the data is not loaded if present, rather data is an open file pointer // if loaded is set then the data is not loaded if present, rather data is an open file pointer // set for reading. typedef struct _track Loading Loading @@ -365,6 +365,24 @@ typedef struct int loaded; // Are arrow vectors loaded in memory? } DAZZ_ARROW; // Every DB is referred to by an ASCII stub file with extension .db or .dam. This file // contains the information about the SMRT cells in the DB and the current division of // the DB into blocks for HPC processing. This file can be read into the following // data structure: typedef struct { int nfiles; // Number of files/SMRT cells in DB int *nreads; // [0..nfiles) = # of reads from cell char **fname; // [0..nfiles) = file name of cell char **prolog; // [0..nfiles) = fasta header prolog for cell int all; // Keep only best read from each well? int cutoff; // Trim reads less than cutoff int64 bsize; // Target size for blocks int nblocks; // Number of blocks for DB int *ublocks; // [0..nblcoks] = index of 1st read in block in untrimmed DB int *tblocks; // [0..nblcoks] = index of 1st read in block in trimmed DB } DAZZ_STUB; // The DB record holds all information about the current state of an active DB including an // array of DAZZ_READS, one per read, and a linked list of DAZZ_TRACKs the first of which // is always a DAZZ_QV pseudo-track (if the QVs have been loaded). Loading Loading @@ -416,6 +434,30 @@ typedef struct #define DB_PARAMS "size = %11lld cutoff = %9d all = %1d\n" // block size, len cutoff, all in well #define DB_BDATA " %9d %9d\n" // First read index (untrimmed), first read index (trimmed) // Read the specified contents of the DB stub file at "path" and return it encoded in a DAZZ_STUB // structure. This is allocated by the routine. "path" is assumed to be the complete // name of the file. If all flags are off, then just the scalar parts of the stub // are returned (i.e. nfiles, all, cutoff, bsize, nblocks). Returns NULL if an error // occured in INTERACTIVE mode #define DB_STUB_NREADS 0x1 #define DB_STUB_FILES 0x2 #define DB_STUB_PROLOGS 0x4 #define DB_STUB_BLOCKS 0x8 DAZZ_STUB *Read_DB_Stub(char *path, int what); // Read the DB stub file "path" and extract the read index range [*first,*last) // for block n, for the trimmed DB if trim is set, the untrimmed DB otherwise. // If n is out of range first and last will be set to -1. Returns 0 unless // an error occurs in INTERACTIVE mode in which case it returns 1. int Fetch_Block_Range(char *path, int trim, int n, int *first, int *last); // Free a DAZZ_STUB data structure returned by Read_DB_Stub void Free_DB_Stub(DAZZ_STUB *stub); /******************************************************************************************* * Loading @@ -428,13 +470,13 @@ typedef struct // (not containing a . !). // A DAM is basically a DB except that: // 1. there are no QV's, instead .coff points the '\0' terminated fasta header of the read // in the file .<dam>.hdr file // 1. there are no QV's, instead .coff points to the '\0' terminated fasta header of the read // in an additional file: .DB.hdr // 2. .origin contains the contig # of the read within a fasta entry (assembly sequences // contain N-separated contigs), and .fpulse the first base of the contig in the // fasta entry // Open the given database or dam, "path" into the supplied DAZZ_DB record "db". If the name has // Open the given database or dam, "path", into the supplied DAZZ_DB record "db". If the name has // a part # in it then just the part is opened. The index array is allocated (for all or // just the part) and read in. // Return status of routine: Loading
DBdump.c +18 −47 Original line number Diff line number Diff line Loading @@ -85,6 +85,7 @@ static int qv_map[51] = int main(int argc, char *argv[]) { DAZZ_DB _db, *db = &_db; DAZZ_STUB *stub; int Quiva_DB, Arrow_DB; int FirstRead; FILE *hdrs = NULL; Loading Loading @@ -291,8 +292,6 @@ int main(int argc, char *argv[]) // If get prolog and file names and index ranges from the .db or .dam file { char *pwd, *root; FILE *dstub; char *dstub_name; int i; if (DAM) Loading @@ -303,41 +302,22 @@ int main(int argc, char *argv[]) if (db->part > 0) *rindex(root,'.') = '\0'; if (DAM) dstub_name = Strdup(Catenate(pwd,"/",root,".dam"),"Allocating dam file name"); stub = Read_DB_Stub(Catenate(pwd,"/",root,".dam"), DB_STUB_NREADS|DB_STUB_FILES|DB_STUB_PROLOGS); else dstub_name = Strdup(Catenate(pwd,"/",root,".db"),"Allocating db file name"); dstub = Fopen(dstub_name,"r"); if (dstub_name == NULL || dstub == NULL) exit (1); stub = Read_DB_Stub(Catenate(pwd,"/",root,".db"), DB_STUB_NREADS|DB_STUB_FILES|DB_STUB_PROLOGS); free(pwd); free(root); FSCANF(dstub,DB_NFILE,&nfiles) fhead = (char **) Malloc(sizeof(char *)*nfiles,"Allocating file list"); ffile = (char **) Malloc(sizeof(char *)*(nfiles+1),"Allocating file list"); findx = (int *) Malloc(sizeof(int *)*(nfiles+1),"Allocating file index"); if (fhead == NULL || ffile == NULL || findx == NULL) exit (1); fhead = stub->prolog; ffile = stub->fname; findx = stub->nreads; nfiles = stub->nfiles; findx += 1; findx[-1] = 0; ffile += 1; ffile[-1] = empty; for (i = 0; i < nfiles; i++) { char prolog[MAX_NAME], fname[MAX_NAME]; FSCANF(dstub,DB_FDATA,findx+i,fname,prolog) if ((fhead[i] = Strdup(prolog,"Adding to file list")) == NULL) exit (1); if ((ffile[i] = Strdup(fname,"Adding to file list")) == NULL) exit (1); } free(dstub_name); fclose(dstub); // If TRIM (the default) then "trim" prolog ranges and the DB if (TRIM) Loading Loading @@ -547,11 +527,11 @@ int main(int argc, char *argv[]) c += 2; } if (map > 0 && findx[map-1] <= b+FirstRead && b+FirstRead < findx[map]) if (map > 0 && findx[map-1] <= b && b < findx[map]) ; else { map = 0; while (b + FirstRead >= findx[map]) while (b >= findx[map]) map += 1; map -= 1; } Loading @@ -566,7 +546,7 @@ int main(int argc, char *argv[]) noreads += 1; if (DOFLN && i+FirstRead >= findx[map]) if (DOFLN && i >= findx[map]) { int ten; if (strcmp(ffile[map+1],ffile[last]) != 0) Loading Loading @@ -595,7 +575,7 @@ int main(int argc, char *argv[]) hdrmax = ten; hdrtot += ten; } else if (i+FirstRead >= findx[map]) else if (i >= findx[map]) { map += 1; ten = strlen(fhead[map]); Loading Loading @@ -713,11 +693,11 @@ int main(int argc, char *argv[]) c += 2; } if (map > 0 && findx[map-1] <= b+FirstRead && b+FirstRead < findx[map]) if (map > 0 && findx[map-1] <= b && b < findx[map]) ; else { map = 0; while (b + FirstRead >= findx[map]) while (b >= findx[map]) map += 1; map -= 1; } Loading @@ -736,7 +716,7 @@ int main(int argc, char *argv[]) flags = r->flags; qv = (flags & DB_QV); if (DOFLN && i+FirstRead >= findx[map]) if (DOFLN && i >= findx[map]) { if (strcmp(ffile[map+1],ffile[last]) != 0) { PRINTF("F %ld %s\n",strlen(ffile[map+1]),ffile[map+1]) last = map+1; Loading @@ -756,7 +736,7 @@ int main(int argc, char *argv[]) PRINTF("L %d %d %d\n",r->origin,r->fpulse,r->fpulse+len) } else { if (i+FirstRead >= findx[map]) { if (i >= findx[map]) { map += 1; PRINTF("H %ld %s\n",strlen(fhead[map]),fhead[map]) } Loading Loading @@ -854,16 +834,7 @@ int main(int argc, char *argv[]) if (DAM) fclose(hdrs); else { int i; for (i = 0; i < nfiles; i++) { free(fhead[i]); free(ffile[i]); } free(fhead); free(ffile-1); free(findx-1); } Free_DB_Stub(stub); Close_DB(db); exit (0); Loading
DBshow.c +14 −41 Original line number Diff line number Diff line Loading @@ -81,12 +81,13 @@ int next_read(File_Iterator *it) int main(int argc, char *argv[]) { DAZZ_DB _db, *db = &_db; DAZZ_STUB *stub; FILE *hdrs = NULL; char *hdrs_name = NULL; int nfiles; char **flist = NULL; int *findx = NULL; char **flist; int *findx; int reps, *pts; int input_pts; Loading Loading @@ -264,44 +265,21 @@ int main(int argc, char *argv[]) if (!DAM) { char *pwd, *root; FILE *dstub; char *dstub_name; int i; root = Root(argv[1],".db"); pwd = PathTo(argv[1]); if (db->part > 0) *rindex(root,'.') = '\0'; dstub_name = Strdup(Catenate(pwd,"/",root,".db"),"Allocating db file name"); dstub = Fopen(dstub_name,"r"); if (dstub_name == NULL || dstub == NULL) exit (1); free(pwd); free(root); FSCANF(dstub,DB_NFILE,&nfiles) flist = (char **) Malloc(sizeof(char *)*nfiles,"Allocating file list"); findx = (int *) Malloc(sizeof(int *)*(nfiles+1),"Allocating file index"); if (flist == NULL || findx == NULL) exit (1); stub = Read_DB_Stub(Catenate(pwd,"/",root,".db"),DB_STUB_NREADS|DB_STUB_PROLOGS); findx += 1; findx[-1] = 0; for (i = 0; i < nfiles; i++) { char prolog[MAX_NAME], fname[MAX_NAME]; FSCANF(dstub,DB_FDATA,findx+i,fname,prolog) if ((flist[i] = Strdup(prolog,"Adding to file list")) == NULL) exit (1); } fclose(dstub); free(dstub_name); nfiles = stub->nfiles; flist = stub->prolog; findx = stub->nreads; // If TRIM (the default) then "trim" prolog ranges and the DB findx[-1] = 0; if (TRIM) { int nid, oid, lid; int cutoff, allflag; Loading @@ -327,8 +305,9 @@ int main(int argc, char *argv[]) } } else if (db->part > 0) { for (i = 0; i < nfiles; i++) else { if (db->part > 0) for (i = 0; i < nfiles; i++) findx[i] -= db->ufirst; } } Loading Loading @@ -660,13 +639,7 @@ int main(int argc, char *argv[]) if (DAM) fclose(hdrs); else { int i; for (i = 0; i < nfiles; i++) free(flist[i]); free(flist); free(findx-1); } Free_DB_Stub(stub); Close_DB(db); exit (0); Loading