maskrcnn example is not training

57 views (last 30 days)
Sam Van Der Jeught
Sam Van Der Jeught on 17 Dec 2021
Hi all,
I'm trying to train a mask r-cnn network on my own data (two classes) and am therefore first trying out the example found on the mathworks website (https://www.mathworks.com/help/deeplearning/ug/instance-segmentation-using-mask-rcnn.html).
The code I'm using (unaltered from the website):
dataFolder = fullfile(tempdir,"coco");
trainedMaskRCNN_url = 'https://www.mathworks.com/supportfiles/vision/data/maskrcnn_object_person_car.mat';
helper.downloadTrainedMaskRCNN(trainedMaskRCNN_url,dataFolder);
pretrained = load(fullfile(dataFolder,'maskrcnn_object_person_car.mat'));
net = pretrained.net;
%%
imTest = imread('visionteam.jpg');
[masks,labels,scores,boxes] = segmentObjects(net,imTest);
overlayedImage = insertObjectMask(imTest,masks);
imshow(overlayedImage)
showShape("rectangle",gather(boxes),"Label",labels,"LineColor",'r')
%%
imageFolder = fullfile(dataFolder,"images");
captionsFolder = fullfile(dataFolder,"annotations");
if ~exist(imageFolder,'dir')
mkdir(imageFolder)
mkdir(captionsFolder)
end
annotationFile = fullfile(captionsFolder,"instances_train2014.json");
str = fileread(annotationFile);
trainClassNames = {'person', 'car'};
numClasses = length(trainClassNames);
imageSizeTrain = [800 800 3];
cocoAPIDir = fullfile(dataFolder,"cocoapi-master","MatlabAPI");
addpath(cocoAPIDir);
unpackAnnotationDir = fullfile(dataFolder,"annotations_unpacked","matFiles");
if ~exist(unpackAnnotationDir,'dir')
mkdir(unpackAnnotationDir)
end
helper.unpackAnnotations(trainClassNames,annotationFile,imageFolder,unpackAnnotationDir);
%%
ds = fileDatastore(unpackAnnotationDir, ...
'ReadFcn',@(x)helper.cocoAnnotationMATReader(x,imageFolder));
dsTrain = transform(ds,@(x)helper.preprocessData(x,imageSizeTrain));
data = preview(dsTrain)
%%
net = maskrcnn("resnet50-coco",trainClassNames,"InputSize",imageSizeTrain)
params = createMaskRCNNConfig(imageSizeTrain,numClasses,[trainClassNames {'background'}]);
params.ClassAgnosticMasks = false;
params.AnchorBoxes = net.AnchorBoxes;
params.FreezeBackbone = true;
initialLearnRate = 0.0012;
momentum = 0.9;
decay = 0.01;
velocity = [];
maxEpochs = 10;
miniBatchSize = 2;
miniBatchFcn = @(img,boxes,labels,masks) deal(cat(4,img{:}),boxes,labels,masks);
mbqTrain = minibatchqueue(dsTrain,4, ...
"MiniBatchFormat",["SSCB","","",""], ...
"MiniBatchSize",miniBatchSize, ...
"OutputCast",["single","","",""], ...
"OutputAsDlArray",[true,false,false,false], ...
"MiniBatchFcn",miniBatchFcn, ...
"OutputEnvironment",["auto","cpu","cpu","cpu"]);
%%
doTraining = true;
if doTraining
iteration = 1;
start = tic;
% Create subplots for the learning rate and mini-batch loss
fig = figure;
[lossPlotter, learningratePlotter] = helper.configureTrainingProgressPlotter(fig);
% Initialize verbose output
helper.initializeVerboseOutput([]);
% Custom training loop
for epoch = 1:maxEpochs
reset(mbqTrain)
shuffle(mbqTrain)
while hasdata(mbqTrain)
% Get next batch from minibatchqueue
[X,gtBox,gtClass,gtMask] = next(mbqTrain);
% Evaluate the model gradients and loss using dlfeval
[gradients,loss,state,learnables] = dlfeval(@networkGradients,X,gtBox,gtClass,gtMask,net,params);
%dlnet.State = state;
% Compute the learning rate for the current iteration
learnRate = initialLearnRate/(1 + decay*(epoch-1));
if(~isempty(gradients) && ~isempty(loss))
[net.AllLearnables,velocity] = sgdmupdate(learnables,gradients,velocity,learnRate,momentum);
else
continue;
end
% Plot loss/accuracy metric every 10 iterations
if(mod(iteration,10)==0)
helper.displayVerboseOutputEveryEpoch(start,learnRate,epoch,iteration,loss);
D = duration(0,0,toc(start),'Format','hh:mm:ss');
addpoints(learningratePlotter,iteration,learnRate)
addpoints(lossPlotter,iteration,double(gather(extractdata(loss))))
subplot(2,1,2)
title(strcat("Epoch: ",num2str(epoch),", Elapsed: "+string(D)))
drawnow
end
iteration = iteration + 1;
end
end
% Save the trained network
modelDateTime = string(datetime('now','Format',"yyyy-MM-dd-HH-mm-ss"));
save(strcat("trainedMaskRCNN-",modelDateTime,"-Epoch-",num2str(epoch),".mat"),'net');
end
When compiling, I get the following error:
Error using networkGradients
Too many output arguments.
Error in deep.internal.dlfeval (line 17)
[varargout{1:nargout}] = fun(x{:});
Error in dlfeval (line 40)
[varargout{1:nargout}] =
deep.internal.dlfeval(fun,varargin{:});
Error in person_car_example (line 80)
[gradients,loss,state,learnables] =
dlfeval(@networkGradients,X,gtBox,gtClass,gtMask,net,params);
so it seems something goes wrong in the line:
[gradients,loss,state,learnables] = dlfeval(@networkGradients,X,gtBox,gtClass,gtMask,net,params);
with the networkGradients file
I have used the networkGradients file from the example folder and have included it below:
function [gradients, totalLoss, state] = networkGradients(X, gTruthBoxes, gTruthLabels, gTruthMasks, dlnet, params)
% networkGradients - Gradient function to train MaskRCNN using a custom
% training loop.
% Copyright 2020 The MathWorks, Inc.
RPNRegDeltas = {'rpnConv1x1BoxDeltas'};regionProposal = {'rpl'};
outputNodes = [ RPNRegDeltas, regionProposal, dlnet.OutputNames(:)'];
% For training, the first step is to run a forward pass on the MaskRCNN
% network.
% We need the following outputs from the network to compute losses -
% YRPNReg - RPN regression output [1x1x4xN] {[deltaX deltaY deltaW deltaH]}
% YRPNClass - RPN classification output [1x1x2xN]
% YRCNNReg - RCNN regression output [1x1x4*(numClasses)xN] {[deltaX deltaY deltaW deltaH]}
% YRCNNClass - RCNN classification output [1x1x(numClasses+1)xN]
% YMask - RCNN Mask segmentation output [hxwxnumClassesxN]
% proposals - The regionproposals from the RPN. [5xN]
[YRPNRegDeltas, proposal, YRCNNClass, YRCNNReg, YRPNClass, YMask, state] = forward(...
dlnet, X, 'Outputs', outputNodes);
% If there are no proposals don't learn anything
if(isempty(proposal))
totalLoss = dlarray([]);
gradients= dlarray([]);
disp("Empty proposals");
return;
end
% Proposals are 5XNumProposals (Due to batch restrictions from custom RPL layer)
proposals = gather(extractdata(proposal));
% Convert proposals to numProposals x 5 (as expected by the rest of post processing code)
proposals =proposals';
proposals(:,1:4) = helper.boxUtils.x1y1x2y2ToXYWH(proposals(:,1:4));
numImagesInBatch = size(gTruthBoxes,1);
%Convert numProposalsx5 Proposals to numImagesInBatchx1 (Group by image index)
proposals = helper.groupProposalsByImageIndex(proposals, numImagesInBatch);
% Generate RCNN response targets
%--------------------------------
% Step 1: Match ground truth boxes to proposals
[assignment, positiveIndex, negativeIndex] = helper.bboxMatchAndAssign(...
proposals, gTruthBoxes,...
params.PositiveOverlapRange, params.NegativeOverlapRange,...
false);
% Step 2: Calcuate regression targets as (dx, dy, log(dw), log(dh))
regressionTargets = helper.generateRegressionTargets(gTruthBoxes, proposals,...
assignment, positiveIndex,...
params.NumClasses);
classNames = categories(gTruthLabels{1});
% Step 3: Assign groundtrutrh labels to proposals
classificationTargets = helper.generateClassificationTargets (gTruthLabels, assignment,...
positiveIndex, negativeIndex,...
classNames, params.BackgroundClass);
% Step 4: Calculate instance weights
instanceWeightsReg = helper.regressionResponseInstanceWeights (classificationTargets, params.BackgroundClass);
% Step 5: Generate mask targets
% Crop and resize the instances based on proposal bboxes and network output size
maskOutputSize = params.MaskOutputSize;
croppedMasks = helper.cropandResizeMasks (gTruthMasks, gTruthBoxes, maskOutputSize);
% Generate mask targets
maskTargets = helper.generateMaskTargets(croppedMasks, assignment, classificationTargets, params);
% Stage 2 (RCNN) Loss
% --------------------
% *Classification loss*
classificationTargets = cat(1, classificationTargets{:})';
% onehotencode labels
classificationTargets = onehotencode(classificationTargets,1);
classificationTargets(isnan(classificationTargets)) = 0;
LossRCNNClass = loss.CrossEntropy(YRCNNClass, classificationTargets);
% *Weighted regression loss*
regressionTargets = cat(1,regressionTargets{:});
instanceWeightsReg = cat(1, instanceWeightsReg{:});
LossRCNNReg = loss.smoothL1(YRCNNReg, single(regressionTargets'), single(instanceWeightsReg'));
% Mask Loss (Weighted cross entropy)
maskTargets= cat(4,maskTargets{:});
positiveIndex = cat(1,positiveIndex{:});
LossRCNNMask = loss.SpatialCrossEntropy(YMask, single(maskTargets), positiveIndex);
% Total Stage 2 loss
LossRCNN = LossRCNNReg + LossRCNNClass + LossRCNNMask;
% Generate RCNN response targets
%--------------------------------
featureSize = size(YRPNRegDeltas);
imageSize = params.ImageSize;
[RPNRegressionTargets, RPNRegWeights, assignedLabelsRPN] = helper.rpnRegressionResponse(featureSize, gTruthBoxes, imageSize, params);
RPNClassificationTargets = onehotencode(assignedLabelsRPN, 3);
RPNClassificationTargets(isnan(RPNClassificationTargets)) = 0;
% Stage 1 (RPN) Loss
% --------------------
LossRPNClass = loss.CrossEntropy(YRPNClass, RPNClassificationTargets);
LossRPNReg = loss.smoothL1(YRPNRegDeltas, RPNRegressionTargets, RPNRegWeights);
LossRPN = LossRPNClass + LossRPNReg;
% Total Loss
%------------
totalLoss = LossRCNN + LossRPN;
gradients = dlgradient(totalLoss, dlnet.Learnables);
end
It seems the "learnables" ouput is one too many, so I remove it but get the following error:
Unrecognized method, property, or field 'OutputNames' for class 'maskrcnn'.
Error in networkGradients (line 10)
outputNodes = [ RPNRegDeltas, regionProposal, dlnet.OutputNames(:)'];
Error in deep.internal.dlfeval (line 17)
[varargout{1:nargout}] = fun(x{:});
Error in dlfeval (line 40)
[varargout{1:nargout}] = deep.internal.dlfeval(fun,varargin{:});
Error in person_car_example (line 80)
[gradients,loss,state] = dlfeval(@networkGradients,X,gtBox,gtClass,gtMask,net,params);
The maskrcnn structure does not have an 'OutputNames' field... It seems likethe entire networkGradients file is incompatible with the example.
Any ideas/help would be very welcome!
Thanks,
Sam

Answers (1)

David Willingham
David Willingham on 20 Dec 2021
Hi Sam,
What version of MATLAB are you running?
David
  1 Comment
Sam Van Der Jeught
Sam Van Der Jeught on 20 Dec 2021
Hi David,
I'm using Matlab 2021b, but I've also tried it with 2020b.
In the meantime I have rewritten the networkGradients.m file as follows and now it works. Hope someone else can benefit from this.
function [gradients, totalLoss, state, learnables] = networkGradients(X, gTruthBoxes, gTruthLabels, gTruthMasks, dlnet, params)
% networkGradients - Gradient function to train MaskRCNN using a custom
% training loop.
% Copyright 2020 The MathWorks, Inc.
%RPNRegDeltas = {'rpnConv1x1BoxDeltas'};regionProposal = {'rpl'};
%outputNodes = [ RPNRegDeltas, regionProposal, dlnet.OutputNames(:)'];
% For training, the first step is to run a forward pass on the MaskRCNN
% network.
% We need the following outputs from the network to compute losses -
% YRPNReg - RPN regression output [1x1x4xN] {[deltaX deltaY deltaW deltaH]}
% YRPNClass - RPN classification output [1x1x2xN]
% YRCNNReg - RCNN regression output [1x1x4*(numClasses)xN] {[deltaX deltaY deltaW deltaH]}
% YRCNNClass - RCNN classification output [1x1x(numClasses+1)xN]
% YMask - RCNN Mask segmentation output [hxwxnumClassesxN]
% % proposals - The regionproposals from the RPN. [5xN]
% [YRPNRegDeltas, proposal, YRCNNClass, YRCNNReg, YRPNClass, YMask, state] = forward(...
% dlnet, X, 'Outputs', outputNodes);
[netOut, state] = forward(dlnet, X);
YRPNClass = netOut{1};
YRPNRegDeltas = netOut{2};
proposal = netOut{3};
YRCNNClass = netOut{4};
YRCNNReg = netOut{5};
YMask = netOut{6};
%save proposal proposal
% If there are no proposals don't learn anything
if(isempty(proposal))
totalLoss = dlarray([]);
gradients= dlarray([]);
disp("Empty proposals");
return;
end
% Proposals are 5XNumProposals (Due to batch restrictions from custom RPL layer)
proposals = gather(extractdata(proposal));
% Convert proposals to numProposals x 5 (as expected by the rest of post processing code)
proposals =proposals';
proposals(:,1:4) = helper.boxUtils.x1y1x2y2ToXYWH(proposals(:,1:4));
numImagesInBatch = size(gTruthBoxes,1);
%Convert numProposalsx5 Proposals to numImagesInBatchx1 (Group by image index)
proposals = helper.groupProposalsByImageIndex(proposals, numImagesInBatch);
% Generate RCNN response targets
%--------------------------------
% Step 1: Match ground truth boxes to proposals
[assignment, positiveIndex, negativeIndex] = helper.bboxMatchAndAssign(...
proposals, gTruthBoxes,...
params.PositiveOverlapRange, params.NegativeOverlapRange,...
false);
%save assignment assignment
% Step 2: Calcuate regression targets as (dx, dy, log(dw), log(dh))
regressionTargets = helper.generateRegressionTargets(gTruthBoxes, proposals,...
assignment, positiveIndex,...
params.NumClasses);
% classNames = categories(gTruthLabels{1}); % !!! TROUBLE, change as below
classNames = params.ClassNames;
% Step 3: Assign groundtrutrh labels to proposals
classificationTargets = helper.generateClassificationTargets (gTruthLabels, assignment,...
positiveIndex, negativeIndex,...
classNames, params.BackgroundClass);
% Step 4: Calculate instance weights
instanceWeightsReg = helper.regressionResponseInstanceWeights (classificationTargets, params.BackgroundClass);
% Step 5: Generate mask targets
% Crop and resize the instances based on proposal bboxes and network output size
maskOutputSize = params.MaskOutputSize;
croppedMasks = helper.cropandResizeMasks (gTruthMasks, gTruthBoxes, maskOutputSize);
% Generate mask targets
maskTargets = helper.generateMaskTargets(croppedMasks, assignment, classificationTargets, params);
% Stage 2 (RCNN) Loss
% --------------------
% *Classification loss*
classificationTargets = cat(1, classificationTargets{:})';
summary(classificationTargets)
% onehotencode labels
classificationTargets = onehotencode(classificationTargets,1);
classificationTargets(isnan(classificationTargets)) = 0;
classificationTargets = reshape(classificationTargets ,1, 1, size(YRCNNClass,3),[]);
LossRCNNClass = loss.CrossEntropy(YRCNNClass, classificationTargets);
% *Weighted regression loss*
regressionTargets = cat(1,regressionTargets{:})';
instanceWeightsReg = cat(1, instanceWeightsReg{:})';
regressionTargets = reshape(regressionTargets, 1, 1, size(YRCNNReg,3),[]);
instanceWeightsReg = reshape(instanceWeightsReg, 1, 1, size(YRCNNReg,3),[]);
LossRCNNReg = loss.smoothL1(YRCNNReg, single(regressionTargets), single(instanceWeightsReg));
% Mask Loss (Weighted cross entropy)
maskTargets= cat(4,maskTargets{:});
positiveIndex = cat(1,positiveIndex{:});
LossRCNNMask = loss.SpatialCrossEntropy(YMask, single(maskTargets), positiveIndex);
% Total Stage 2 loss
LossRCNN = LossRCNNReg + LossRCNNClass + LossRCNNMask;
% Generate RCNN response targets
%--------------------------------
featureSize = size(YRPNRegDeltas);
imageSize = params.ImageSize;
[RPNRegressionTargets, RPNRegWeights, assignedLabelsRPN] = helper.rpnRegressionResponse(featureSize, gTruthBoxes, imageSize, params);
RPNClassificationTargets = onehotencode(assignedLabelsRPN, 3);
% detetron2 uses ony foreground class for classification.
RPNClassificationTargets(:,:,2,:) = [];
RPNClassificationTargets = reshape(RPNClassificationTargets, featureSize(1), featureSize(2), [], numImagesInBatch );
RPNClassificationTargets(isnan(RPNClassificationTargets)) = 0;
% Stage 1 (RPN) Loss
% --------------------
YRPNClass = softmax(YRPNClass);
LossRPNClass = loss.CrossEntropy(YRPNClass, RPNClassificationTargets);
LossRPNReg = loss.smoothL1(YRPNRegDeltas, RPNRegressionTargets, RPNRegWeights);
LossRPN = LossRPNClass + LossRPNReg;
% Total Loss
%------------
totalLoss = LossRCNN + LossRPN;
if(params.FreezeBackbone)
learnables = rmfield(dlnet.AllLearnables, {'FeatureExtractionNet','PostPoolFeatureExtractionNet'});
else
learnables = dlnet.AllLearnables;
end
gradients = dlgradient(totalLoss, learnables);
end

Sign in to comment.

Community Treasure Hunt

Find the treasures in MATLAB Central and discover how the community can help you!

Start Hunting!

Translated by