clear all
close all
clc
%% input
% csvDir = 'p:\11205259-004-dcsm-fm\waterlevel_data\RWS_data-distributielaag\rawData\';
% csvFiles = dir([csvDir 'data_*' filesep 'BROUWHVSGT08_OW_WATHTE_NVT_ddl_wq_20220328.csv']);
csvDir = 'p:\11202493--systeemrap-grevelingen\1_data\Noordzee\ddl\raw\wathte\';
csvFiles = dir([csvDir '*.csv']);
ncDir = 'p:\11202493--systeemrap-grevelingen\1_data\Noordzee\ddl\netcdf\wathte\';

addpath('n:\Deltabox\Bulletin\groenenb\hydro_tools\data_conversion_scripts\')

encPliDCSM = 'p:\1204257-dcsmzuno\data\RWS_data-distributielaag\saliniteit_temperatuur\script\DCSM-FM_0_5nm_grid_20200211.pli';
pol = io_polygon('read',encPliDCSM);

%%
allStationNames = strtok({csvFiles.name},'_');
stationNames = unique(allStationNames);
for iS = 1:length(stationNames)
    stationName = stationNames{iS};
    csvInd = strmatch(stationName,allStationNames,'exact');
    ncFile = [ncDir  filesep stationName '.nc'];
    
%     if exist(ncFile,'file')
%         continue
%     end

    for iF = 1:length(csvInd)
        disp(['status: ' num2str(iS) '/' num2str(length(stationNames)) ' - ' num2str(iF) '/' num2str(length(csvFiles(csvInd))) ])
        ind = csvInd(iF);
        csvFile = [csvFiles(ind).folder filesep csvFiles(ind).name];
        
        fid = fopen(csvFile,'r');
        data1 = textscan(fid,'%s','delimiter','\n');
        data1 = data1{1};
        % data2(1:length(data1),1:length(strsplit(data1{1},';'))) = {''}; % allocate
        data2 = repmat({''},length(data1),length(strsplit(data1{1},';')));
        for i = 1:length(data1)
            data2(i,:) = strsplit(data1{i},';');
        end
        fclose(fid);
        
        if iF == 1
            data3 = data2;
        else
            data3 = [data3; data2(2:end,:)]; % skip header
        end
    end
    
    header = data3(1,:);
    data3(1,:) = [];
    
    clear A
    
    ind = strmatch('numeriekewaarde',header,'exact');
    A.waterlevel = cellfun(@str2num,data3(:,ind))/100;
    A.waterlevel(A.waterlevel > 999) = NaN; % fix because, apparently, dummy value is not always exactly 999999999?
    [A.waterlevel,idSort] = sort(A.waterlevel);
    data3 = data3(idSort,:);

    indS = strmatch('statuswaarde',header,'exact');
    [~,idSort] = sort(data3(:,indS));
    data3 = data3(idSort,:);
    A.waterlevel = A.waterlevel(idSort);
    
    ind = strmatch('tijdstip',header,'exact');
    t = data3(:,ind);
    t = cell2mat(t);
    t = t(:,[1:4 6:7 9:10 12:13 15:16]);
    [A.time,idSort] = sort(datenum(t,'yyyymmddHHMM'));
    data3 = data3(idSort,:);
    A.waterlevel = A.waterlevel(idSort);
    
    % Idea to speed this up: sort data3 by value (absolute low to high),
    % sort data3 by status (A-Z) --> first value is most plausible and
    % 'Gecontroleerd' --> remove all second duplicate times       
    if ~isempty(indS)
        if 1
            [A.time, idUni] = unique(A.time,'stable');
            A.waterlevel    = A.waterlevel(idUni);
            data3           = data3(idUni,:);
        else
        [~, idUni]  = unique(A.time,'stable');
        idDuplicate = setdiff(1:numel(A.time),idUni);
        disp(['Filtering ' num2str(length(idDuplicate)) ' duplicates based on statuswaarde'])
        idDel = [];
        for iD = idDuplicate
            idSet = find(A.time==A.time(iD));
            if strcmp(data3(idSet(1),indS),'Ongecontroleerd')
                idDel = [idDel;idSet(1)];
            elseif strcmp(data3(idSet(2),indS),'Ongecontroleerd')
                idDel = [idDel;idSet(2)];
            elseif isnan(A.waterlevel(idSet(1)))
                idDel = [idDel;idSet(1)];
            elseif isnan(A.waterlevel(idSet(2)))
                idDel = [idDel;idSet(2)];
            else
                warning('No difference in status and value, deleted first duplicate')
                idDel = [idDel;idSet(1)];
            end
        end
        A.time(idDel)        = [];
        A.waterlevel(idDel)  = [];
        data3(idDel,:)       = [];
        end
    end
        
    % platform_id/_name
    ind = strmatch('locatie.code',header,'exact');
    A.platform_id = data3{1,ind};
    ind = strmatch('locatie.naam',header,'exact');
    A.platform_name = data3{1,ind};
    
    % x/y-coordinaat
    ind = strmatch('geometriepunt.x',header,'exact'); if length(unique(data3(:,ind))) > 1; error; end
    if ~isempty(ind)
        x = str2num(data3{1,ind});
        ind = strmatch('geometriepunt.y',header,'exact');
        y = str2num(data3{1,ind});
        ind = strmatch('coordinatenstelsel',header,'exact');
        [A.lon,A.lat] = convertCoordinates(x,y,'CS1.code',str2num(data3{1,ind}),'CS2.code',4326);
        
        if ~inpolygon(A.lon,A.lat,pol(:,1),pol(:,2))
            disp(['Skipping station ' A.platform_name '. Data outside model domain'])
            continue
        end
    else
        A.lon = 999;
        A.lat = 999;
    end
       
    A.originalDataFiles = csvDir;
        
    %save to nc file
    if exist(ncFile,'file')
        delete(ncFile);
    end
    write_waterleveldata_nc(A,ncFile);
    disp(['created file: ' ncFile])
end
