Efficient import of semi structured text

Question

I have multiple text files saved from a Tekscan pressure mapping system. I'm am trying to find the most efficient method for importing the multiple comma delimited matrices into one 3-d matrix of type uint8. I have developed a solution, which makes repeated calls to the MATLAB function dlmread. Unfortunately, it takes roughly 1.5 min to import the data. I have included the code below.

This code makes calls to two other functions I wrote, metaextract and framecount which I have not included as they aren't truly relevant to answering the question at hand.

Here are two links to samples of the files I am using.

The first is a shorter file with 90 samples

The second is a longer file with 3458 samples

Any help would be appreciated

function pressureData = tekscanimport
% Import TekScan data from .asf file to 3d matrix of type double.

[id,path] = uigetfile('*.asf'); %User input for .asf file
if path == 0 %uigetfile returns zero on cancel
    error('You must select a file to continue')
end

path = strcat(path,id); %Concatenate path and id to full path

% function calls
pressureData.metaData = metaextract(path);
nLines = linecount(path); %Find number of lines in file
nFrames = framecount(path,nLines);%Find number of frames

rowStart = 25; %Default starting row to read from tekscan .asf file
rowEnd = rowStart + 41; %Frames are 42 rows long
colStart = 0;%Default starting col to read from tekscan .asf file
colEnd = 47;%Frames are 48 rows long
pressureData.frames = zeros([42,48,nFrames],'uint8');%Preallocate for speed

f = waitbar(0,'1','Name','loading Data...',...
    'CreateCancelBtn','setappdata(gcbf,''canceling'',1)');
setappdata(f,'canceling',0);

for i = 1:nFrames %Loop through file skipping frame metadata
    if getappdata(f,'canceling')
        break
    end
    waitbar(i/nFrames,f,sprintf('Loaded %.2f%%', i/nFrames*100));

    %Make repeated calls to dlmread
    pressureData.frames(:,:,i) = dlmread(path,',',[rowStart,colStart,rowEnd,colEnd]);
    rowStart = rowStart + 44;
    rowEnd = rowStart + 41;
end
delete(f)
end

Answer 1

I gave it a try. This code opens your big file in 3.6 seconds on my PC. The trick is to use sscanf instead of the str2double and str2number functions.

clear all;tic
fid = fopen('tekscanlarge.txt','rt');
%read the header, stop at frame
header='';
l = fgetl(fid);
while length(l)>5&&~strcmp(l(1:5),'Frame')
    header=[header,l,sprintf('\n')];
    l = fgetl(fid);
    if length(l)<5,l(end+1:5)=' ';end
end
%all data at once
dat = fread(fid,inf,'*char');
fclose(fid);
%allocate space
res = zeros([48,42,3458],'uint8');
%get all line endings
LE = [0,regexp(dat','\n')];
i=1;
for ct = 2:length(LE)-1 %go line by line
    L = dat(LE(ct-1)+1:LE(ct)-1);
    if isempty(L),continue;end
    if all(L(1:5)==['Frame']')
        fr = sscanf(L(7:end),'%u');
        i=1;
        continue;
    end
    % sscan can only handle row-char with space seperation.
    res(:,i,fr) = uint8(sscanf(strrep(L',',',' '),'%u'));
    i=i+1;
end
toc

Does anyone knows of a faster way to convert than sscanf? Because it spends the majority of time on this function (2.17 seconds). For a dataset of 13.1MB I find it very slow compared to the speed of the memory.

Found a way to do it in 0.2 seconds that might be usefull for others as well. This mex-file scans through a list of char values for numbers and reports them back. Save it as mexscan.c and run mex mexscan.c.

#include "mex.h" 
/* The computational routine */
void calc(unsigned char *in, unsigned char *out, long Sout, long Sin)
{
    long ct = 0;
    int newnumber=0; 
    for (int i=0;i<Sin;i+=2){
        if (in[i]>=48 && in[i]<=57) { //it is a number
            out[ct]=out[ct]*10+in[i]-48;
            newnumber=1;
        } else { //it is not a number
            if (newnumber==1){
                ct++;
                if (ct>Sout){return;}
            }
            newnumber=0;
        }
    }    
}

/* The gateway function */
void mexFunction( int nlhs, mxArray *plhs[], int nrhs, const mxArray *prhs[])
{
    unsigned char *in;             /* input vector */
    long Sout;                     /* input size of output vector */
    long Sin;                      /* size of input vector */
    unsigned char *out;            /* output vector*/

    /* check for proper number of arguments */
    if(nrhs!=2) {
        mexErrMsgIdAndTxt("MyToolbox:arrayProduct:nrhs","two input required.");
    }
    if(nlhs!=1) {
        mexErrMsgIdAndTxt("MyToolbox:arrayProduct:nlhs","One output required.");
    }
    /* make sure the first input argument is type char */
    if(!mxIsClass(prhs[0], "char"))  {
        mexErrMsgIdAndTxt("MyToolbox:arrayProduct:notDouble","Input matrix must be type char.");
    }
    /* make sure the second input argument is type uint32 */
    if(!mxIsClass(prhs[0], "char"))  {
        mexErrMsgIdAndTxt("MyToolbox:arrayProduct:notDouble","Input matrix must be type char.");
    }

    /* get dimensions of the input matrix */
    Sin = mxGetM(prhs[0])*2;
    /* create a pointer to the real data in the input matrix  */
    in = (unsigned char *) mxGetPr(prhs[0]);
    Sout = mxGetScalar(prhs[1]);

    /* create the output matrix */
    plhs[0] = mxCreateNumericMatrix(1,Sout,mxUINT8_CLASS,0);

    /* get a pointer to the real data in the output matrix */
    out = (unsigned char *) mxGetPr(plhs[0]);

    /* call the computational routine */
    calc(in,out,Sout,Sin);
}

Now this script runs in 0.2 seconds and returns the same result as the previous script.

clear all;tic
fid = fopen('tekscanlarge.txt','rt');
%read the header, stop at frame
header='';
l = fgetl(fid);
while length(l)>5&&~strcmp(l(1:5),'Frame')
    header=[header,l,sprintf('\n')];
    l = fgetl(fid);
    if length(l)<5,l(end+1:5)=' ';end
end
%all data at once
dat = fread(fid,inf,'*char');
fclose(fid);
S=[48,42,3458];
d = mexscan(dat,uint32(prod(S)+3458));
d(1:prod(S(1:2))+1:end)=[];%remove frame numbers
d = reshape(d,S);
toc