getVIS =1
  getNIR =0
  getVNIR=1

 ;--I
  root = '/data/spectroscopy/smass/'
  date = '2017-08-29'

  dir=root+date+'/'
  cata = 'spectracatalog-'+date+'.csv' 
  csv = 'smass.mit.edu-'+date+'.csv' 

 ;--I
;  dir = '/home/bcarry/data/mining/smass/'
;  cata = 'spectracatalog-2016-05-18.csv' 
;  csv = 'smass.mit.edu-2016-05-19.csv' 

 ;--II--Read list
  readcol, dir+cata, delimiter=',', num, name, des, vis, nir, visnir, format='(I,A,A,A,A,A)',/Silent
  nbSmass = n_elements( des )
;  print, nbSmass

 ;--III--Read HTML
  openr, id, dir+csv, /get_lun


  line=''
  kL=0L

  spawn, 'wc '+dir+csv+" -l|awk '{print $1}'", nbLine
  sso = replicate({num:0L, name:'', des:'', wave:'',link:'', ref:'', ep:''},nbLine)


  while ~eof(id) do begin

    readf, id, line
    split = strSplit( line, ',', /Extract, count=nbField )


    case nbField of

       9: begin
          pos = strpos( split[0], ')' )
          if pos ne -1 then begin
            sub = strSplit( split[0],'()',/Extract)
            sso[kL].num = round(float(sub[1]))
            sso[kL].name= strTrim(split[1],2)
          endif else begin
            sso[kL].num = sso[kL-1].num 
            sso[kL].name= sso[kL-1].name
            sso[kL].des = sso[kL-1].des
          endelse

          sso[kL].wave = strTrim( split[2],2)
          sso[kL].link = strTrim( split[3],2)
          sso[kL].ref  = strTrim( split[7],2)
          sso[kL].ep   = strTrim( split[8],2)
       end

       7: begin
          pos = strpos( split[0], ')' )
          if pos ne -1 then begin
            sub = strSplit( split[0],'()',/Extract)
            sso[kL].num = round(float(sub[1]))
            sso[kL].name= strTrim(split[1],2)
          endif else begin
            sso[kL].num = sso[kL-1].num 
            sso[kL].name= sso[kL-1].name
            sso[kL].des = sso[kL-1].des
          endelse

          sso[kL].wave = strTrim( split[2],2)
          sso[kL].link = strTrim( split[3],2)
          sso[kL].ref  = strTrim( split[5],2)
          sso[kL].ep   = strTrim( split[6],2)
       end
      

       10: begin
          pos = strpos( split[0], ')' )
          if pos ne -1 or strcmp( strtrim(split[0],2),'0') then begin

            if pos ne -1 then begin
              sub = strSplit( split[0],'()',/Extract)
              sso[kL].num = round(float(sub[1]))
            endif
            sso[kL].name= strTrim(split[1],2)
            sso[kL].des = strTrim(split[2],2)
          endif else begin
            sso[kL].num = sso[kL-1].num 
            sso[kL].name= sso[kL-1].name
            sso[kL].des = sso[kL-1].des
          endelse

          sso[kL].wave = strTrim( split[3],2)
          sso[kL].link = strTrim( split[4],2)
          sso[kL].ref  = strTrim( split[8],2)
          sso[kL].ep   = strTrim( split[9],2)
          if strcmp( strtrim(split[0]),'' ) then stop
       end

       8: begin
          pos = strpos( split[0], ')' )
          if pos ne -1 or strcmp( strtrim(split[0],2),'0') then begin

            if pos ne -1 then begin
              sub = strSplit( split[0],'()',/Extract)
              sso[kL].num = round(float(sub[1]))
            endif
            sso[kL].name= strTrim(split[1],2)
            sso[kL].des = strTrim(split[2],2)
          endif else begin
            sso[kL].num = sso[kL-1].num 
            sso[kL].name= sso[kL-1].name
            sso[kL].des = sso[kL-1].des
          endelse

          sso[kL].wave = strTrim( split[3],2)
          sso[kL].link = strTrim( split[4],2)
          sso[kL].ref  = strTrim( split[6],2)
          sso[kL].ep   = strTrim( split[7],2)

       end

       11: begin
          pos = strpos( split[0], ')' )
          if pos ne -1 or strcmp( strtrim(split[0],2),'0') then begin

            if pos ne -1 then begin
              sub = strSplit( split[0],'()',/Extract)
              sso[kL].num = round(float(sub[1]))
            endif
            sso[kL].name= strTrim(split[1],2)
            sso[kL].des = strTrim(split[2],2)
          endif else begin
            sso[kL].num = sso[kL-1].num 
            sso[kL].name= sso[kL-1].name
            sso[kL].des = sso[kL-1].des
          endelse

          sso[kL].wave = strTrim( split[3],2)
          sso[kL].link = strTrim( split[4],2)
          sso[kL].ref  = strTrim( split[8],2)
          sso[kL].ep   = strTrim( split[10],2)

       end

       else: begin
       end
    endcase


    kL++
  endwhile


  close, id
  free_lun, id


  sel = where( strcmp(sso.name, '') )
  sso[sel].name = sso[sel].des


  forprint, sso.num, sso.name, sso.wave, sso.ref, sso.link, textout=dir+'info.smass.csv', $
            format='(I6," , ",A-20," , ",A-7," , ",A-8," , ",A-60)', $
            comment='Number, Name, Waverange, Reference, Link', /Silent


  if getNIR eq 1 then begin
    selNIR = where( strCmp(sso.wave,'NIR'), nbNIR )
    for kNIR=0, nbNIR-1 do begin
  
      split = strsplit(sso[selNIR[kNIR]].name,' ',/extract, count=nbField)
      if nbField eq 1 then name = sso[selNIR[kNIR]].name else $
         name = split[0]+'_'+split[1]
      spawn, 'curl -s "'+sso[selNIR[kNIR]].link+'" > '+dir+'NIR/'+name+'.txt'
  
    endfor
  endif
  if getVNIR eq 1 then begin
    selVISNIR = where( strCmp(sso.wave,'Vis+NIR'), nbVISNIR )
    for kVISNIR=0, nbVISNIR-1 do begin
  
      split = strsplit(sso[selVISNIR[kVISNIR]].name,' ',/extract, count=nbField)
      if nbField eq 1 then name = sso[selVISNIR[kVISNIR]].name else $
         name = split[0]+'_'+split[1]
      spawn, 'curl -s "'+sso[selVISNIR[kVISNIR]].link+'" > '+dir+'VisNIR/'+name+'.txt'
  
    endfor
  endif

  if getVis eq 1 then begin
    selVIS = where( strCmp(sso.wave,'Vis'), nbVIS )
    for kVIS=0, nbVIS-1 do begin
  
      split = strsplit(sso[selVIS[kVIS]].name,' ',/extract, count=nbField)
      if nbField eq 1 then name = sso[selVIS[kVIS]].name else $
         name = split[0]+'_'+split[1]
      spawn, 'curl -s "'+sso[selVIS[kVIS]].link+'" > '+dir+'Vis/'+name+'.txt'
  
    endfor
  endif


end