diff --git a/Data/SandingImagesCalibration_Min.mat b/Data/SandingImagesCalibration_Min.mat new file mode 100644 index 0000000..4377d00 Binary files /dev/null and b/Data/SandingImagesCalibration_Min.mat differ diff --git a/Data/testImage1.bmp b/Data/testImage1.bmp new file mode 100644 index 0000000..dba9280 Binary files /dev/null and b/Data/testImage1.bmp differ diff --git a/Data/testImage13.bmp b/Data/testImage13.bmp new file mode 100644 index 0000000..322d223 Binary files /dev/null and b/Data/testImage13.bmp differ diff --git a/Dependency_Analysis.m b/Dependency_Analysis.m new file mode 100644 index 0000000..7654aaf --- /dev/null +++ b/Dependency_Analysis.m @@ -0,0 +1,19 @@ +%% Run the Toolbox Dependency Analysis +fileList = {}; + +moduleFilesStruct = dir('Modules'); +moduleFiles = arrayfun(@(x) [x.folder '\' x.name],moduleFilesStruct,'UniformOutput',false); + +mainFilesStruct = dir; +mainFiles = arrayfun(@(x) x.name, mainFilesStruct, 'UniformOutput',false); + +for i = 1:length(moduleFiles) + if(endsWith(moduleFiles{i} ,'.m')); fileList{end+1} = moduleFiles{i}; end; +end + +for i = 1:length(mainFiles) + if(endsWith(mainFiles{i} ,'.m')); fileList{end+1} = mainFiles{i}; end; +end + +toolBoxDependencies = dependencies.toolboxDependencyAnalysis(fileList); +dependencies.toolboxDependencyAnalysis({'RESClassifier.m'}) \ No newline at end of file diff --git a/Modules/Calibration.m b/Modules/Calibration.m new file mode 100644 index 0000000..eb77502 --- /dev/null +++ b/Modules/Calibration.m @@ -0,0 +1,326 @@ +% ========================================================================= +% === Calibration Package for Automated Surface Inspection Integration === +% ========================================================================= +% Author: Christian Schirmer (christian.schirmer@uconn.edu) +% ========================================================================= + +classdef Calibration < handle + + properties + % calData stores region bitmasks, along with corresponding pose and + % section identifiers + calData = table( ... + 'Size',[0 3],... + 'VariableTypes',{'double', 'double', 'cell'},... + 'VariableNames',{'PoseID','SectionID','Bitmask'}) + + % calImages stores calibration images. They are stored as key-value + % pairs --> + calImages = containers.Map('KeyType','double','ValueType','any'); + end + + % User-Accessible Calibration Functions + methods (Access = public) + function this = Calibration(varargin) + % CALIBRATION Calibration object constructor. Data will be loaded + % from filename argument, if specified. + if(nargin == 0) + % No input argument: Do nothing, proceed normally. + elseif(nargin ==1) + % 1 input argument: Filename. Load data from this file. + this.loadData(varargin{1}); + end + end + + function addPose(this, PoseID, image_filepath) + % ADDPOSE: Adds a pose using a calibration image. + % A pose is added with the specified PoseID and calibration image + % at image_filepath. The user is prompted to draw regions around + % the different sections to create the pose-to-section mapping. + + % If the pose already exists in the data table, ask the user if + % they would like to replace the pose + + % Check to see if the pose already exists in the table + if(any(this.calData.PoseID == PoseID) | this.calImages.isKey(PoseID)) + userInput = questdlg(... + ['Pose ' num2str(PoseID) ' has already been added. ' ... + 'All calibration data associated with this pose ' ... + 'will be erased. Would you like to continue?'], ... + ['Overwrite Pose' num2str(PoseID) '?'], ... + 'Yes','No','No'); + switch userInput + case 'Yes' + this.removePose(PoseID); + otherwise + return; + end + end + + % Read the calibration image in from the specified filepath + this.calImages(PoseID) = imread(image_filepath); + + % Now we go through the region-drawing procedure to create + % bitmasks for all the sections in the calibration image. + sectionsComplete = false; + while(~sectionsComplete) + try + this.addRegion(PoseID); + catch MExc + if(strcmp(MExc.identifier,'Calibration:addRegion:drawROIWindowClosed')) + % This error is thrown when the user closes the UI + % for adding a region before actually drawing the + % region. This indicates the user would like to + % stop adding regions. + sectionsComplete = true; + elseif(strcmp(MExc.identifier,'Calibration:addRegion:sectionInputCancelled')) + % User Cancelled SectionID input. Do nothing. + % Section will not be saved. User will be + % re-prompted to draw section. + else + % Any other errors are not expected and should be + % re-thrown. + rethrow(MExc); + end + end + end + end + + function removePose(this, PoseID) + % REMOVEPOSE: Removes all data associated with the specified Pose. + % This includes the calibration image, and all regions associated + % with the pose. + + % Remove calibration image + this.calImages.remove(PoseID); + % Remove all regions associated with the specified pose + this.calData = this.calData((this.calData.PoseID~=PoseID),:); + end + + function addRegion(this, PoseID) + % ADDREGION: Prompts the user to draw a region in the + % calibration image for the given pose. Prompts user for section + % number. Generates bitmap and appends to calibration data. + + % === Retrieve Existing Bitmasks for this PoseID === + % Get all bitmasks associated with that PoseID + bm_list = this.getBitmasks(PoseID); + % Flatten all the bitmasks on top of each other for display, collect + % information for labeling: section centroid, label text + bm_centroids = {}; + bm_labels = {}; + for i = 1:height(bm_list) + % Retrieve the i'th bitmask + bitmask_i = logical(bm_list{i,'Bitmask'}{1}); + if(i==1) + % First bitmask will not need to be flattened. Assign it as + % totalMask + totalMask = bitmask_i; + else + % Flatten current bitmask into totalMask + totalMask = logical(totalMask) | bitmask_i; + end + + % Find Centroid of Region, Save SectionID as Label + % Reference: https://www.mathworks.com/matlabcentral/answers/322369-find-centroid-of-binary-image + [grid_y, grid_x] = ndgrid(1:size(totalMask,1),1:size(totalMask,2)); + bm_centroids{i} = mean([grid_x(logical(bitmask_i)),grid_y(logical(bitmask_i))]); + + % Store SectionID as label + bm_labels{i} = num2str(bm_list{i,'SectionID'}); + end + + % === Draw the Figure for the User to Draw a Region on === + + % Have the user define a region by drawing a polygon + roi = images.roi.Polygon; + % Create a figure + thisFig = figure; + % Show the calibration image + curCalImage = imshow(this.calImages(PoseID)); + title('Please draw boundary around a section (or close window if complete)'); + + % If there are existing bitmasks, show them + if(height(bm_list)>0) + hold on; + % Create a green mask layer to display over sections + maskLayer = cat(3,zeros(size(totalMask)),ones(size(totalMask)),zeros(size(totalMask))); + % Draw the already-defined bitmasks on top of the image + maskIm = imshow(maskLayer); + % Label the masks + for i = 1:length(bm_centroids) + text(bm_centroids{i}(1),bm_centroids{i}(2),bm_labels{i}); + end + hold off; + + % Set the mask layer transparency to 20% + set(maskIm,'AlphaData',totalMask*.2); + end + + + % Add the ROI drawing tool to the figure so the user can draw + % a region around the next subsection + draw(roi); + + % Check to see if window closed + if(~isvalid(roi)) + throw(MException('Calibration:addRegion:drawROIWindowClosed','Draw ROI window closed before ROI was drawn')); + end + + % Generate the Bitmask + bitmask = createMask(roi, curCalImage); + + % Prompt the user for a section number + userInput = inputdlg(... + 'SectionID associated with boundary:',... + 'Enter SectionID'); + if(size(userInput)==[0 0]) % Check to see if user has cancelled input + delete(thisFig); + throw(MException('Calibration:addRegion:sectionInputCancelled','User cancelled section input')); + end + secID = str2num(userInput{1}); + + % Check to see if the section already exists in the calibration + % data. User can Replace, Merge, or Cancel. + if(any(... + (this.calData.SectionID == secID) & ... + (this.calData.PoseID == PoseID))... + ) + userInput = questdlg(... + ['Pose ' num2str(PoseID) ' already has a region ' ... + 'defined for section ' num2str(secID) '. Would you' ... + ' like to replace the existing region, merge in ' ... + ' the new selection, or leave the existing region' ... + ' as-is?'], ... + ['Region Conflict: Pose' num2str(PoseID) ', Section'... + num2str(secID)], ... + 'Replace','Merge','Cancel', 'Replace'); + switch userInput + case 'Replace' + % Remove the existing bitmask entry for this section, + % replace with the new one. + this.removeCalData(PoseID, secID); + this.appendCalData(PoseID, secID, bitmask); + case 'Merge' + % Merge the existing bitmask with the new one. + old_bm = this.calData(this.calData.PoseID==PoseID & this.calData.SectionID==secID,:).Bitmask{1}; + this.calData{this.calData.PoseID==PoseID & this.calData.SectionID==secID,'Bitmask'}= ... + {(old_bm | bitmask)}; + case 'Cancel' + % Do nothing. + otherwise + % User input closed. Do nothing. + end + else + % Save the data back to calData table + this.appendCalData(PoseID, secID, bitmask); + end + + % Close the figure + delete(thisFig); + end + + function save(this, filename) + % SAVE: Saves a .mat file containing the full Calibration object + builtin('save',filename, 'this'); + end + + function bitmasks = getBitmasks(this, PoseID) + % GETBITMASKS: Get a table of Bitmasks and SectionIDs for a given PoseID + bitmasks = this.calData((this.calData.PoseID==PoseID),{'SectionID', 'Bitmask'}); + end + + function viewPose(this, PoseID) + % VIEWPOSE: Displays the calibration image for the specified pose, + % as well as all regions that have been associated with the pose. + figure; + imshow(this.getPoseImage(PoseID)); + title(['Current Calibration: Pose ' num2str(PoseID)]); + + end + end + + % Private 'Helper' Functions + methods (Access = private) + function this = appendCalData(this, PoseID, SectionID, bitmask) + % Appends a row to the calData table + this.calData = [this.calData; {PoseID, SectionID, {bitmask}}]; + end + + function removeCalData(this, PoseID, SectionID) + % Remove a row from the calData table (row(s) corresponding to the + % given PoseID and SectionID + this.calData = this.calData(this.calData.PoseID~=PoseID | this.calData.SectionID ~= SectionID,:); + end + + function loadData(this,filename) + % Loads saved data from a .mat file into the Calibration object + fileData = load(filename); + this.calData = fileData.this.calData; + this.calImages = fileData.this.calImages; + end + + function compositeImage = getPoseImage(this, PoseID) + + % === Retrieve Existing Bitmasks for this PoseID === + % Get all bitmasks associated with that PoseID + bm_list = this.getBitmasks(PoseID); + % Flatten all the bitmasks on top of each other for display, collect + % information for labeling: section centroid, label text + bm_centroids = {}; + bm_labels = {}; + for i = 1:height(bm_list) + % Retrieve the i'th bitmask + bitmask_i = logical(bm_list{i,'Bitmask'}{1}); + if(i==1) + % First bitmask will not need to be flattened. Assign it as + % totalMask + totalMask = bitmask_i; + else + % Flatten current bitmask into totalMask + totalMask = logical(totalMask) | bitmask_i; + end + + % Find Centroid of Region, Save SectionID as Label + % Reference: https://www.mathworks.com/matlabcentral/answers/322369-find-centroid-of-binary-image + [grid_y, grid_x] = ndgrid(1:size(totalMask,1),1:size(totalMask,2)); + bm_centroids{i} = mean([grid_x(logical(bitmask_i)),grid_y(logical(bitmask_i))]); + + % Store SectionID as label + bm_labels{i} = num2str(bm_list{i,'SectionID'}); + end + + % Retrieve the calibration image associated with the pose + calImage = this.calImages(PoseID); + + % Define the transparency of the bitmasks over the calibration + % image + alpha = .3; + + % Create a transparency map: alpha transparency within the + % bitmap region, full transparency everywhere else. + alpha_map = (alpha*totalMask); + + % Create a green-only layer + green_image = 255*cat(3,zeros(size(totalMask)),ones(size(totalMask)),zeros(size(totalMask))); + + % Convert the bitmask to a green RGB image + % bm_image = uint8(cat(3,zeros(size(totalMask)),totalMask*255,zeros(size(totalMask)))); + + % Paint the transparent bitmask over the calibration image + % Reference: https://en.wikipedia.org/wiki/Alpha_compositing + compositeImage = uint8((green_image).*alpha_map + double(calImage).*(1-alpha_map)); + + + % Add the section labels to the image. + compositeImage = insertText(compositeImage,... + reshape(cell2mat(bm_centroids),2,length(bm_centroids))',... + bm_labels,... + 'AnchorPoint', 'Center',... + 'FontSize',min(floor(size(compositeImage,2)/40),200)); + + end + + end + +end \ No newline at end of file diff --git a/Modules/Func_Capture_Image.m b/Modules/Func_Capture_Image.m new file mode 100644 index 0000000..9fca5fc --- /dev/null +++ b/Modules/Func_Capture_Image.m @@ -0,0 +1,27 @@ +function img = Func_Capture_Image() +%FUNC_CAPTURE_IMAGE Implementation of image capture from the camera. +% This function should return an n x m x 3 uint8 image array + + test = 1; + + if(~test) + % Initialize a connection to the camera + g = gigecam('169.254.90.219','PixelFormat', 'BayerBG8'); + + % Change the ExposureTime setting (in us) + g.ExposureTimeAbs = 20000; + + % Acquire a single image from the camera + img = snapshot(g); + + % Clean up by clearing the object. + clear g; + else + % Just return a test image + img = imread('./Data/testImage13.bmp'); + end + + + +end + diff --git a/Modules/Func_Process_Images.m b/Modules/Func_Process_Images.m new file mode 100644 index 0000000..3d839b7 --- /dev/null +++ b/Modules/Func_Process_Images.m @@ -0,0 +1,73 @@ +function [sectionStatus] = Func_Process_Images(imageContainer) +%FUNC_PROCESS_IMAGES Process the images and returns status of each section +% 0 = Good Part (default, if no image given: log event) +% 1 = Bad Part + +% Step 0: Load Calibration & Evaluation Algorithm +c = Calibration('./Data/SandingImagesCalibration_Min.mat'); +classifier_data = load('./Data/RES101TrainedNET.mat'); +classifier_net = classifier_data.net; + +fprintf('%s Loading Images...\n', datestr(now,'HH:MM:SS.FFF')) +% Step 1: Load all the images +im_PoseID = []; +images = {}; + +i = 0; +for keys = imageContainer.keys + i=i+1; + im_PoseID(i) = keys{1}; + images{i} = imageContainer(keys{1}); +end + +fprintf('%s Masking and Cropping Images...\n', datestr(now,'HH:MM:SS.FFF')) +% Step 2: Mask and Crop all the images +% +masked_images = {}; +cropped_images = {}; +im_mask_PoseID = []; +im_mask_SecID = []; +i = 0; +for j = 1:length(images) + masks = c.calData{c.calData{:,'PoseID'}==im_PoseID(j),'Bitmask'}; + secIDs = c.calData{c.calData{:,'PoseID'}==im_PoseID(j),'SectionID'}; + for k = 1:length(masks) + i = i+1; + im_mask_PoseID(i) = im_PoseID(j); + im_mask_SecID(i) = secIDs(k); + % == Mask Image == + masked_images{i} = images{j}.*uint8(masks{k}); + % == Crop Image == + im = sum(masked_images{i},3); + % Flatten in horizontal and vertical dimensions, find uppermost and + % lowermost nonzero rows and columns + dim1 = sum(im,1); + dim2 = sum(im,2); + dim1_idx = [find(dim2,1,'first'),find(dim2,1,'last')]; + dim2_idx = [find(dim1,1,'first'),find(dim1,1,'last')]; + cropped_images{i} = masked_images{i}(dim1_idx(1):dim1_idx(2),dim2_idx(1):dim2_idx(2),:); + end +end + + +fprintf('%s Evaluating Images...\n', datestr(now,'HH:MM:SS.FFF')) +% Step 3: Evaluate all the images +% Initialize section status array. Index corresponds to PoseID now. +sectionStatus = logical(zeros(1,max(im_PoseID))); +checkedStatus = logical(zeros(1,max(im_PoseID))); +for i = 1:length(im_mask_PoseID) + [Predicted_Class, elapsed_prediction_time] = ... + RESClassifier(classifier_net,cropped_images{i}); + % Class 1 = Not Well Sanded (Bad Part) | Class 2 = Well Sanded (Good Part) + sectionStatus(im_mask_SecID(i)) = sectionStatus(im_mask_SecID(i)) || (Predicted_Class == '1'); + % + checkedStatus(im_mask_SecID(i)) = 1; +end +fprintf('%s Evaluation Complete.\n', datestr(now,'HH:MM:SS.FFF')) + + + + + +end + diff --git a/Modules/Func_Send_Capture_Complete.m b/Modules/Func_Send_Capture_Complete.m new file mode 100644 index 0000000..75c9d7b --- /dev/null +++ b/Modules/Func_Send_Capture_Complete.m @@ -0,0 +1,13 @@ +function Func_Send_Capture_Complete(tcpConn) +%FUNC_SEND_ROBOT_MSG Signals the robotics system that the image has been +%captured. + +bits = [true logical(zeros(1,63))]; + +% Convert bits to a uint8 (byte) array +msg = uint8(bi2de(reshape(bits,8,[])','left-msb'))'; + +% Send the message +fwrite(tcpConn,msg,'uint8'); +end + diff --git a/Modules/Func_Send_Section_Statuses.m b/Modules/Func_Send_Section_Statuses.m new file mode 100644 index 0000000..edf1661 --- /dev/null +++ b/Modules/Func_Send_Section_Statuses.m @@ -0,0 +1,17 @@ +function Func_Send_Section_Statuses(tcpConn, sectionStatusRegister) +%FUNC_SEND_SECTION_STATUSES Sends sections statuses to robotics system + + +% Right-Pad the section status register to a length of 63, and left-pad +% with one zero (the camera done bit) +bits = [false, sectionStatusRegister, logical(zeros(1,63-length(sectionStatusRegister)))]; + +% Convert bits to a uint8 (byte) array +msg = uint8(bi2de(reshape(bits,8,[])','left-msb'))'; + +% Send the message +fwrite(tcpConn,msg,'uint8'); + + +end + diff --git a/RESClassifier.m b/RESClassifier.m new file mode 100644 index 0000000..7033458 --- /dev/null +++ b/RESClassifier.m @@ -0,0 +1,23 @@ +function [Predict_Class, TestTime] = RESClassifier(net, testImage) +%% Create New Image Classification + +%% load Test Image and performace preprocessing +% +Inputsize = net.Layers(1).InputSize; +Resize_testImage = imresize(testImage,Inputsize(1,1:2)); + +if length(size(Resize_testImage)) == 3 + Color_resize_testImage = Resize_testImage; +elseif length(size(Resize_testImage)) == 2 + Color_resize_testImage = cat(3,Resize_testImage,Resize_testImage,Resize_testImage); +end + +%% Classify Classification +% YPred is the class number the model predict +% TestTime is the amount of time the model takes to predict +tic; +Predict_Class = classify(net,Color_resize_testImage); +TestTime = toc; + +end + diff --git a/StateFlowChart.sfx b/StateFlowChart.sfx new file mode 100644 index 0000000..1d45039 Binary files /dev/null and b/StateFlowChart.sfx differ diff --git a/main.m b/main.m new file mode 100644 index 0000000..8ca5af2 --- /dev/null +++ b/main.m @@ -0,0 +1,42 @@ +%% Setup Settings +% Load Modules +addpath('./Modules'); +% Integrated Robot Controller System Connection Configuration: +setup = struct(); +setup.IRC_IP_Address = 'localhost'; % IP Address of IRC +setup.IRC_IP_Port = 60451; % Connection Port on IRC +setup.IRC_Packet_Terminator = 'CR'; % Carriage Return + +%% Load the Finite State Machine +fsm = StateFlowChart(); + +%% Begin TCP Communication with the IRC +tcpConn = tcpip(setup.IRC_IP_Address,setup.IRC_IP_Port) +fsm.dataStore.tcpConn = tcpConn; +fsm.dataStore.tcpSetup = setup; +tcpConn.BytesAvailableFcn = {@msgRcv,fsm}; % Edit the callback function here +tcpConn.BytesAvailableFcnMode = 'terminator'; +tcpConn.Terminator = setup.IRC_Packet_Terminator; + +fopen(tcpConn); + +%% Use a timer to step the fsm +tmr = timer('ExecutionMode','fixedDelay'); +tmr.TimerFcn = {@stepFSM,fsm}; +tmr.start; + +%% Callback Functions + +function stepFSM(obj,event,fsm) + fsm.step; +end + +function msgRcv(obj,event,fsm) +% msgRcv: Event handler for receiving data from the robot + + % Read data from the buffer + msgData = fread(obj,obj.BytesAvailable); + % Pass the relevant bytes along (last byte is message terminator) + msgHandler(msgData(1:end-1),fsm); + +end \ No newline at end of file diff --git a/msgHandler.m b/msgHandler.m new file mode 100644 index 0000000..7659802 --- /dev/null +++ b/msgHandler.m @@ -0,0 +1,36 @@ +function msgHandler(msgData,fsm) +%MSGHANDLER Handles Robot-To-PC Messages +% Robot-To-PC Status Packet +% 0-Bit | 1-Bit | Name +% 1(LSB) Robot Ready Bit +% 2 Start Inspection Bit +% 3 Inspection Complete Bit + + % Convert messaage bytes to bit array + msgBits = reshape(de2bi(uint8(msgData),'left-msb')',[],1)'; + + % Convert to bytes + msgByte = uint8(msgData(1)); + + %% Trigger events based on the incoming message data + + % Robot Ready Bit (Capture Image) + if(bitand(uint8(1),msgByte,'uint8')) + % Pass the current PoseID to the FSM + fsm.curPoseID = bi2de(msgBits(3:10),'left-msb'); + fsm.ev_Req_Image_Capture(); + end + + % Inspection Complete Bit + if(bitand(uint8(4),msgByte)) + fsm.ev_Req_Complete_Inspection(); + end + + % Start Inspection Bit + if(bitand(uint8(2),msgByte)) + fsm.ev_Req_Begin_Inspection(); + end + + +end +