function [dpnames,errorMsg] = getCOVISdp(deviceid,startdate,enddate,searchDetails,isPostProcess)
%%---------------------------------------------------------------------
% General:  Generate COVIS dp (mat and png) files from covis tar source files
% Input:    deviceid  (usually 19003)
%           startdate (datenum)
%           enddate (datenum)
%           searchDetails
%           isPostProcess - true if we only want to process one file (the first), produce both png and mat files and name them for archiving
% Output:   dpnames
%           errorMsg
% Notes:    Assumes one mat/png file for each tar file. Here is an example searchDetails structure:
%             searchDetails.searchid=-1;
%             searchDetails.networkid=NaN;
%             searchDetails.resultid=1;
%             searchDetails.dataproductformatid = 72; %(70=imaging mat, 71=imaging png, 72=diffuse mat, 73=imaging png)
%             searchDetails.dataproductid = 41;  %(40=imaging, 41=diffuse)
%             searchDetails.archivefileextensionid = 9;  %(9=mat, 2=png)
%           An example call is:
%             getCOVISdp(19003,735061,735062,searchDetails,0)
%           With the searchDetails as specified above, this will make diffuse mat files for an entire day.
% Modify Date/Author:  
%           20120705, B.Biffard 
%---------------------------------------------------------------------
%% check on the inputs
if nargin<1
    deviceid=19003; 
end
if nargin<2
    startdate=floor(now)-datenum([0 0 1 0 0 0]);
end
if nargin<3
    enddate=floor(now); 
end   
if nargin<4
    searchDetails.searchid=-1;
    searchDetails.networkid=NaN;
    searchDetails.dataproductformatid=70;
    searchDetails.dataproductid=40;
    searchDetails.archivefileextensionid=9;
    searchDetails.resultid=1;
end
if nargin<5
    isPostProcess = false;
end

%% set up some variables
matfile = '';
imgfile = '';
errorMsg = ''; 
dpnames = {};
subsampleFactor = 1;  %we can decimate the data. I've tested this and it works, however it is SLOWER than using the non-decimated data.
%flag to clean up all of the non-DP files - true on the task machines
if any(getEnv==[2 4])
    doCleanUp = false;    
else
    doCleanUp = true;
end

%% search AD for the files - bit of trick to seperate them
archivePath = getPathAD(getEnvData); 
if searchDetails.dataproductid == 40
    adfiles = getADfiles(deviceid,startdate,enddate,'%IMAGING.tar');
    if isempty(adfiles)
        disp(['AD file fetch found no *IMAGING.tar files for deviceID = ' num2str(deviceid) ', date >= '...
            datestr(startdate,'yyyy-mm-dd HH:MM:SS') ', date < ' datestr(enddate,'yyyy-mm-dd HH:MM:SS')]);
        return;
    end
elseif searchDetails.dataproductid == 41
    adfiles = getADfiles(deviceid,startdate,enddate,'%DIFFUSE.tar');
    if isempty(adfiles)
        disp(['AD file fetch found no *DIFFUSE.tar files for deviceID = ' num2str(deviceid) ', date >= '...
            datestr(startdate,'yyyy-mm-dd HH:MM:SS') ', date < ' datestr(enddate,'yyyy-mm-dd HH:MM:SS')]);
        return;
    end
else
    error('NEPTUNE:getCOVISdp','input dpID must be 40 or 41');
end

%% pull in accessory files by detecting the path to SVN and copying the entire directory over to the working directory - not nessecary
% the covis code assumes this directory is contained within the current working directory. So it's safest to copy it over.
% However, if matlab can't find the file locally, it will search up the path to find it. So as long as the input directory is in the path,
% we don't need to copy it over. So I can safely comment out this code:
% if exist('input','dir')>0
%     ctmp = which('covisMatlabPostProcessJob.m','-all');
%     pathC = fileparts(ctmp{1});
%     copyfile([pathC filesep 'input' filesep '*'],'input')
% end

%% if this is a post process job, select the file that is closest to the start date and just process that file
if isPostProcess
    adfiles = adfiles(1);
end

%% loop over the files, run the covis code
for iFile = 1:length(adfiles);
    try            
        %get the start and end dates of the current source file
        [~,n,~]=fileparts(adfiles{iFile});
        ndi = strfind(n,'-'); 
        n = n(1:ndi(end)-1); %trim off the ancillary info and extra extensions in the case of .tar.gz
        nui = strfind(n,'_'); 
        if length(nui) < 2
            nui(2) = length(n)+1;
        end
        [~,sd,~] = fileparts(n(nui(1)+1:nui(2)-1));
        sd = datenum(sd,'yyyymmddTHHMMSS');
        ed = sd + 2/24;  %covis scans are three hours apart and last about 45 minutes - this should be fetched from the database.
        %cap the start and end dates to the search dates
        sd = max(sd, startdate);
        ed = min(ed, enddate);

        if ~isPostProcess
            if searchDetails.dataproductid == 40 && searchDetails.archivefileextensionid == 9
                fString = '%IMAGING.mat';
            elseif searchDetails.dataproductid == 40 && searchDetails.archivefileextensionid == 2
                fString = '%IMAGING.png';
            elseif searchDetails.dataproductid == 41 && searchDetails.archivefileextensionid == 9
                fString = '%DIFFUSE.mat';
            elseif searchDetails.dataproductid == 41 && searchDetails.archivefileextensionid == 2
                fString = '%DIFFUSE.png';
            else
                error('NEPTUNE:getCOVISdp','input searchDetails must be dataproductID 40 or 41 and archiveExtension 2 or 9');
            end
            % if dp file exists already in AD for the dt4, then simply retrieve from AD,
            % otherwise generate mat file on-the-fly. Only accept the first file, one DP file per tar file.
            adDPfiles = getADfiles(deviceid,sd,ed,fString);
            if length(adDPfiles) > 0 %#ok<ISMT> %files exist in archive for the current source file (assume complete)     
                if length(adDPfiles) > 1
                    warning('NEPTUNE:getCOVISdp','too many dp files returned in search - one dp file per tar file expected');
                end
                if searchDetails.searchid>0
                    modifyMainSearchResultEntryComment(searchDetails.resultid,['Retrieving pre-processed ' fString(2:end) ' file from the archive, for source file # ' num2str(iFile) ' of ' num2str(length(adfiles)) '.'])
                end
                url = [archivePath adDPfiles{1}];
                [~, bn, ext] = fileparts(adDPfiles{1});
                di = strfind(bn,'-');
                %REQS-1400 - add .gz extension and dateto to the file-name
                adDPname = [bn(1:di(1)-1) '_' getADfilesDateTo(deviceid, adDPfiles{1}, ext) '.000Z'...
                    bn(di(1):end) ext getADfilesCompressExt(deviceid,adDPfiles{1})];
                [~,status] = urlwrite(url,adDPname); 
                %append filename to output and continue to the next tar file, if the file was retreived from AD.
                if status && (exist(adDPname,'file')>0)
                    dpnames = [dpnames; {adDPname}];
                    continue;
                end 
            end
        end       
        
        %if we don't hit the continue statement, we're processing on the fly        
        if searchDetails.searchid>0
            modifyMainSearchResultEntryComment(searchDetails.resultid,['Running... working on source file # ' num2str(iFile) ' of ' num2str(length(adfiles)) '.'])
        end

        %get the meta data for plot titles and mat files, using the startdate of the current file
        metadata = getDeviceMeta(deviceid,sd);              

        % if the file doesn't exist, get it from AD, AD files are always .gz
        % if the the extension isn't .gz, try adding a .gz to the filename
        if ~exist(adfiles{iFile},'file')
            if ~exist([adfiles{iFile} '.gz'],'file')
                url = [archivePath adfiles{iFile}];
                [~,~,e] = fileparts(adfiles{iFile});
                if ~strcmp(e,'.gz')
                    adfiles{iFile} = [adfiles{iFile} '.gz'];
                end                    
                urlwrite(url,adfiles{iFile});
            else 
                adfiles{iFile} = [adfiles{iFile} '.gz'];                
            end
        end

        %run the covis code - decimate 
        if subsampleFactor > 1
            extractpathname = covis_decimate(adfiles{iFile}, cd, subsampleFactor);
        else
            extractpathname = adfiles{iFile};
        end

        %decide between IMAGING or DIFFUSE data products
        if strfind(adfiles{iFile},'IMAGING')
            matfile = covis_imaging_sweep(extractpathname, cd, '');
            if searchDetails.archivefileextensionid == 2 || isPostProcess
                imgfile = covis_imaging_plot(matfile, cd, '');
            end
            typeString = 'IMAGING';
        elseif strfind(adfiles{iFile},'DIFFUSE')   
            [swp_path, swp_name] = covis_extract(extractpathname, 0);
            [c_struct, matfile] = covis_diffuse_sweep(swp_path, swp_name,'');
            if (searchDetails.archivefileextensionid == 2 || isPostProcess) && ~isempty(c_struct)
                [~, ~, imgfile] = covis_diffuse_plot(swp_path, c_struct, '');
            end
            typeString = 'DIFFUSE';
        else
            error('NEPTUNE:covisMatlabPostProcess',['Cannot determine if the given COVIS file ' adfiles{iFile} ' is a IMAGING or DIFFUSE file']);
        end     

        dateto = '';
        if length(regexp(adfiles{iFile},'_........T.*Z','end')) == 1                
            dateto = getADfilesDateTo(deviceid,adfiles{iFile});
        end

        if (searchDetails.archivefileextensionid == 2 || isPostProcess) && ~isempty(imgfile)
            %trim off the path
            [~,in,ie] = fileparts(imgfile);
            imgfile = [in ie];

            %add the titles to the plots
            titleString = getCOVISPlotTitle(metadata, startdate, typeString);
            titleImageName = makeTitleImage(2, titleString);
            img = imread(imgfile);
            timg = imread(titleImageName);
            if ~all(size(img) == size(timg))
                error('NEPTUNE:covisMatlabPostProcessJob','Title image and plot are not the same size.');
            end
            tHeight = ceil(0.08*size(img,1));
            bHeight = ceil(0.025*size(img,1));
            img([1:tHeight (size(img,1)-bHeight):end],:,:) = timg([1:tHeight (size(img,1)-bHeight):end],:,:);
            imwrite(img,imgfile);                

            %gather the names of the resulting data products and rename them for post processing
            if ~isempty(date) && isPostProcess
                iei = regexp(imgfile,'_........T.*Z','end');
                imgfileFull = [imgfile(1:iei) '_' dateto '.000Z' imgfile(iei+1:end)];                
                dpnames = [dpnames; {imgfileFull}];       
                movefile(imgfile,imgfileFull);
            else
                dpnames = [dpnames; {imgfile}];         
            end                
        end

        if (searchDetails.archivefileextensionid == 9 || isPostProcess) && ~isempty(matfile)
            %trim off the path
            [~,mn,me] = fileparts(matfile);
            matfile = [mn me];

            %add the metadata to the mat files
            load(matfile);
            save(matfile,'covis','metadata');

            %gather the names of the resulting data products and rename them for post processing
            if ~isempty(dateto) && isPostProcess
                mei = regexp(matfile,'_........T.*Z','end');
                matfileFull = [matfile(1:mei) '_' dateto '.000Z' matfile(mei+1:end)];              
                dpnames = [dpnames; {matfileFull}];           
                movefile(matfile,matfileFull);  
            else
                dpnames = [dpnames; {matfile}];           %#ok<*AGROW>
            end                
        end
    catch ME
        if exist('ME','var')
            errorMsg = [errorMsg ' MATLAB error in getCOVISdp.m, the problem has been logged and will be addressed shortly.'];  %#ok<AGROW>
            disp(' ')
            disp(getReport(ME))
            disp(' ')
        end        
    end
    %Inform user of bad data        
    if searchDetails.searchid>0 && ((isempty(imgfile) && searchDetails.archivefileextensionid == 9) || (isempty(matfile) && searchDetails.archivefileextensionid == 2))
        modifyMainSearchResultEntryComment(searchDetails.resultid,['Warning... source file # ' num2str(iFile) ' of ' num2str(length(adfiles)) ' appears to be corrupted or empty.'])
    end
end        

%% clean up
if doCleanUp
    fList = dir;
    for f = 1:length(fList)
        if all([~any(strcmp(dpnames,fList(f).name)) ~strcmp('.',fList(f).name) ~strcmp('..',fList(f).name) ~strcmp('.svn',fList(f).name)]) 
            if exist(fList(f).name,'dir') > 0
                rmdir(fList(f).name,'s');
            else
                if exist(fList(f).name,'file') > 0
                    delete(fList(f).name)
                end
            end
        end
    end
end

