[37] | 1 | % * This code was used in the following articles:
|
---|
| 2 | % * [1] Learning 3-D Scene Structure from a Single Still Image,
|
---|
| 3 | % * Ashutosh Saxena, Min Sun, Andrew Y. Ng,
|
---|
| 4 | % * In ICCV workshop on 3D Representation for Recognition (3dRR-07), 2007.
|
---|
| 5 | % * (best paper)
|
---|
| 6 | % * [2] 3-D Reconstruction from Sparse Views using Monocular Vision,
|
---|
| 7 | % * Ashutosh Saxena, Min Sun, Andrew Y. Ng,
|
---|
| 8 | % * In ICCV workshop on Virtual Representations and Modeling
|
---|
| 9 | % * of Large-scale environments (VRML), 2007.
|
---|
| 10 | % * [3] 3-D Depth Reconstruction from a Single Still Image,
|
---|
| 11 | % * Ashutosh Saxena, Sung H. Chung, Andrew Y. Ng.
|
---|
| 12 | % * International Journal of Computer Vision (IJCV), Aug 2007.
|
---|
| 13 | % * [6] Learning Depth from Single Monocular Images,
|
---|
| 14 | % * Ashutosh Saxena, Sung H. Chung, Andrew Y. Ng.
|
---|
| 15 | % * In Neural Information Processing Systems (NIPS) 18, 2005.
|
---|
| 16 | % *
|
---|
| 17 | % * These articles are available at:
|
---|
| 18 | % * http://make3d.stanford.edu/publications
|
---|
| 19 | % *
|
---|
| 20 | % * We request that you cite the papers [1], [3] and [6] in any of
|
---|
| 21 | % * your reports that uses this code.
|
---|
| 22 | % * Further, if you use the code in image3dstiching/ (multiple image version),
|
---|
| 23 | % * then please cite [2].
|
---|
| 24 | % *
|
---|
| 25 | % * If you use the code in third_party/, then PLEASE CITE and follow the
|
---|
| 26 | % * LICENSE OF THE CORRESPONDING THIRD PARTY CODE.
|
---|
| 27 | % *
|
---|
| 28 | % * Finally, this code is for non-commercial use only. For further
|
---|
| 29 | % * information and to obtain a copy of the license, see
|
---|
| 30 | % *
|
---|
| 31 | % * http://make3d.stanford.edu/publications/code
|
---|
| 32 | % *
|
---|
| 33 | % * Also, the software distributed under the License is distributed on an
|
---|
| 34 | % * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either
|
---|
| 35 | % * express or implied. See the License for the specific language governing
|
---|
| 36 | % * permissions and limitations under the License.
|
---|
| 37 | % *
|
---|
| 38 | % */
|
---|
| 39 | function [matches fail] = MatchPointsGivenOcclusion( defaultPara, ImgInfo1, ImgInfo2, ImgScale1, ImgScale2, Img1, Img2, Img1Index, Img2Index, ... |
---|
| 40 | Pair, GlobalScale, Wrlname, PostFixStrAfter,... |
---|
| 41 | POriReprojM1, FieldOccluPix1, FaceSetPickedIND1, ... |
---|
| 42 | POriReprojM2, FieldOccluPix2, FaceSetPickedIND2, FlagEarlyStopMatchPointsGivenOcclusion) |
---|
| 43 | % initialize parameters |
---|
| 44 | displayFlag = 0; |
---|
| 45 | fail = 0; |
---|
| 46 | RefineCorrSpace = 10; |
---|
| 47 | depthratioMin = 0.01; |
---|
| 48 | depthratioMax = 100; |
---|
| 49 | |
---|
| 50 | if nargin < 20 |
---|
| 51 | FlagEarlyStopMatchPointsGivenOcclusion = 0; |
---|
| 52 | end |
---|
| 53 | % 1)Prepare the Constrain to run the matching |
---|
| 54 | |
---|
| 55 | % load surf Features |
---|
| 56 | [f1] = readSurf(Img1, defaultPara.Fdir, 'Dense'); % original features |
---|
| 57 | [f2] = readSurf(Img2, defaultPara.Fdir, 'Dense'); % original features |
---|
| 58 | |
---|
| 59 | % initialize the Rc ConS ConSRough |
---|
| 60 | NumSurF1 = length(f1); |
---|
| 61 | NumSurF2 = length(f2); |
---|
| 62 | Rc1 = [ ones(1,NumSurF1); zeros(1,NumSurF1); ones(1,NumSurF1); zeros(1,NumSurF1)]; |
---|
| 63 | Rc2 = [ ones(1,NumSurF2); zeros(1,NumSurF2); ones(1,NumSurF2); zeros(1,NumSurF2)]; |
---|
| 64 | ConS1 = zeros(4,NumSurF1); |
---|
| 65 | ConS2 = zeros(4,NumSurF2); |
---|
| 66 | ConSRough1 = ConS1; |
---|
| 67 | ConSRough2 = ConS2; |
---|
| 68 | ConS1_4points = zeros(8,NumSurF1); |
---|
| 69 | ConS2_4points = zeros(8,NumSurF2); |
---|
| 70 | AllPOriReprojM1 = zeros(2,NumSurF1); |
---|
| 71 | AllPOriReprojM2 = zeros(2,NumSurF2); |
---|
| 72 | AllFieldOccluPix1 = zeros(2,NumSurF1); |
---|
| 73 | AllFieldOccluPix2 = zeros(2,NumSurF2); |
---|
| 74 | |
---|
| 75 | % calculate constrain for effective points |
---|
| 76 | if ~isempty(FaceSetPickedIND1) |
---|
| 77 | [ Rc1(:,FaceSetPickedIND1) ConS1(:,FaceSetPickedIND1) ConSRough1(:,FaceSetPickedIND1) ConS1_4points(:,FaceSetPickedIND1)] = ... |
---|
| 78 | EndPoint2BoxConS(defaultPara, ImgScale1(1), ImgScale1(2), POriReprojM1, FieldOccluPix1, 1); |
---|
| 79 | AllPOriReprojM1(:,FaceSetPickedIND1) = POriReprojM1; |
---|
| 80 | AllFieldOccluPix1(:,FaceSetPickedIND1) = FieldOccluPix1; |
---|
| 81 | end |
---|
| 82 | if ~isempty(FaceSetPickedIND2) |
---|
| 83 | [ Rc2(:,FaceSetPickedIND2) ConS2(:,FaceSetPickedIND2) ConSRough2(:,FaceSetPickedIND2) ConS2_4points(:,FaceSetPickedIND2)] = ... |
---|
| 84 | EndPoint2BoxConS(defaultPara, ImgScale2(1), ImgScale2(2), POriReprojM2, FieldOccluPix2, 1); |
---|
| 85 | AllPOriReprojM2(:,FaceSetPickedIND2) = POriReprojM2; |
---|
| 86 | AllFieldOccluPix2(:,FaceSetPickedIND2) = FieldOccluPix2; |
---|
| 87 | end |
---|
| 88 | |
---|
| 89 | T1_hat = [[0 -Pair.T(3) Pair.T(2)];... |
---|
| 90 | [Pair.T(3) 0 -Pair.T(1)];... |
---|
| 91 | [-Pair.T(2) Pair.T(1) 0]]; |
---|
| 92 | F = inv(defaultPara.InrinsicK2)'*T1_hat*Pair.R*inv(defaultPara.InrinsicK1); |
---|
| 93 | I1=imreadbw([defaultPara.Fdir '/pgm/' Img1 '.pgm']); % function from sift |
---|
| 94 | I2=imreadbw([defaultPara.Fdir '/pgm/' Img2 '.pgm']); % function from sift |
---|
| 95 | if displayFlag |
---|
| 96 | |
---|
| 97 | figure; |
---|
| 98 | dispMatchSearchRegin(I1, I2, [f1; ones(1,NumSurF1)], [f2; ones(1,NumSurF2)], ConS1_4points, ConS2_4points, F, ... |
---|
| 99 | AllPOriReprojM1, ones(1,NumSurF1), AllFieldOccluPix1, ones(1,NumSurF1), ... |
---|
| 100 | AllPOriReprojM2, ones(1,NumSurF2), AllFieldOccluPix2, ones(1,NumSurF2), ... |
---|
| 101 | 1, 'Stacking', 'h', 'Interactive', 0); |
---|
| 102 | end |
---|
| 103 | |
---|
| 104 | if ~( isempty(FaceSetPickedIND1)&&isempty(FaceSetPickedIND2)) % only if not both FaceSetPickedIND is empty then find the matches |
---|
| 105 | % write the constrain into data |
---|
| 106 | Vector2Ipoint([Rc1; ConS1],[defaultPara.Fdir '/surf/'],['RConS_' Img1]); |
---|
| 107 | Vector2Ipoint([Rc2; ConS2],[defaultPara.Fdir '/surf/'],['RConS_' Img2]); |
---|
| 108 | Vector2Ipoint([ConSRough1],[defaultPara.Fdir '/surf/'],['RConSRough_' Img1]); |
---|
| 109 | Vector2Ipoint([ConSRough2],[defaultPara.Fdir '/surf/'],['RConSRough_' Img2]); |
---|
| 110 | |
---|
| 111 | %======================= debug only |
---|
| 112 | % save([defaultPara.Fdir '/data/PreOcclusionDetect.mat'],'Rc1','ConS1','Rc2','ConS2','ConSRough1','ConSRough2'); |
---|
| 113 | % return; |
---|
| 114 | %================================== |
---|
| 115 | |
---|
| 116 | % run time consuming matching code |
---|
| 117 | tic; |
---|
| 118 | cd match |
---|
| 119 | system(['./surfOccluMatch.sh ' defaultPara.Fdir ' ' Img1 ' ' Img2 ' OccluDense ' '0.2 0.6']); % Parameter still need to be changed//Min |
---|
| 120 | cd .. |
---|
| 121 | toc |
---|
| 122 | |
---|
| 123 | % Readin matching result |
---|
| 124 | [f1, f2, matches] = readSurfMatches(Img1, Img2, defaultPara.Fdir, [ defaultPara.Type 'OccluDense'], 1, 1, 3); |
---|
| 125 | % in 'OccluDense' cases matches is N by 3, the last column is Ratio |
---|
| 126 | if isempty( matches) |
---|
| 127 | disp( 'Zeros Surf Occlusion Matches'); |
---|
| 128 | fail = 1; |
---|
| 129 | return; |
---|
| 130 | end |
---|
| 131 | matches = matches(1:2,:); |
---|
| 132 | |
---|
| 133 | else |
---|
| 134 | % no matches, means no need to storage new ConstrainOccluMatch |
---|
| 135 | matches = []; |
---|
| 136 | end |
---|
| 137 | |
---|
| 138 | if displayFlag |
---|
| 139 | figure(200); plotmatches(I1,I2,f1, f2,matches, 'Stacking','v','Interactive', 3); |
---|
| 140 | saveas(200,[ defaultPara.ScratchFolder Img1 '_' Img2 '_OccluMatches'],'jpg'); |
---|
| 141 | end |
---|
| 142 | |
---|
| 143 | if FlagEarlyStopMatchPointsGivenOcclusion |
---|
| 144 | matches = [f1(:,matches(1,:)); f2(:,matches(2,:))]; |
---|
| 145 | save([defaultPara.Fdir '/data/' Img1 '_' Img2 '_' PostFixStrAfter '.mat'],'f1','f2','matches'); |
---|
| 146 | return; |
---|
| 147 | end |
---|
| 148 | |
---|
| 149 | % 2) Process the matches |
---|
| 150 | |
---|
| 151 | % Pruning by epipolarline |
---|
| 152 | if ~isempty(matches) |
---|
| 153 | [inlier Residual] = EpipoPrune(defaultPara, Pair, [f1(:,matches(1,:)); f2(:,matches(2,:))], (ImgScale1+ImgScale2)/2); |
---|
| 154 | matches = matches(:,inlier); |
---|
| 155 | |
---|
| 156 | if defaultPara.Flag.FlagCorrRefinement |
---|
| 157 | % Fineer Search of the close by the SurfMatches Features by Corrolation Matches ========Min Added July 13th |
---|
| 158 | if ~isempty(Pair.lamda) |
---|
| 159 | depthratio = Pair.lamda(1,:)./Pair.lamda(2,:); |
---|
| 160 | % Min add to remove outliers (should be already removed when doing PoseEst.m) |
---|
| 161 | Inliers = depthratio > depthratioMin & depthratio < depthratioMax; |
---|
| 162 | depthratio = depthratio(Inliers); |
---|
| 163 | maxRatio = max(depthratio); |
---|
| 164 | minRatio = min(depthratio); |
---|
| 165 | else |
---|
| 166 | maxRatio = max([GlobalScale(1)/GlobalScale(2) GlobalScale(2)/GlobalScale(1)]); |
---|
| 167 | minRatio = 1/maxRatio; |
---|
| 168 | end |
---|
| 169 | |
---|
| 170 | % construction epipolar line unit vector |
---|
| 171 | EpipolarUnitVector1 = F*[ f1(:,matches(1,:)); ones(1,size(matches,2))]; |
---|
| 172 | EpipolarUnitVector2 = F'*[ f2(:,matches(2,:)); ones(1,size(matches,2))]; |
---|
| 173 | EpipolarUnitVector1 = EpipolarUnitVector1([2 1],:); |
---|
| 174 | EpipolarUnitVector2 = EpipolarUnitVector2([2 1],:); |
---|
| 175 | EpipolarUnitVector1 = EpipolarUnitVector1./repmat( sqrt( sum( EpipolarUnitVector1.^2, 1)), 2, 1); |
---|
| 176 | EpipolarUnitVector2 = EpipolarUnitVector2./repmat( sqrt( sum( EpipolarUnitVector2.^2, 1)), 2, 1); |
---|
| 177 | |
---|
| 178 | [Matches1 CoeffM1 Inliers1]=CorrolationMatch( defaultPara, Pair, I1, I2, f1(:,matches(1,:)), ... |
---|
| 179 | f2(:,matches(2,:)) + EpipolarUnitVector1*RefineCorrSpace, ... |
---|
| 180 | f2(:,matches(2,:)) - EpipolarUnitVector1*RefineCorrSpace, [minRatio maxRatio],[1 2]); |
---|
| 181 | Pair2_1.R = Pair.R'; |
---|
| 182 | Pair2_1.T = -Pair.R*Pair.T; |
---|
| 183 | [Matches2 CoeffM2 Inliers2]=CorrolationMatch( defaultPara, Pair2_1, I2, I1, f2(:,matches(2,:)), ... |
---|
| 184 | f1(:,matches(1,:)) + EpipolarUnitVector2*RefineCorrSpace, ... |
---|
| 185 | f1(:,matches(1,:)) - EpipolarUnitVector2*RefineCorrSpace, [minRatio maxRatio],[1 2]); |
---|
| 186 | Matches1 = Matches1(:,Inliers1); |
---|
| 187 | Matches2 = Matches2(:,Inliers2); |
---|
| 188 | CoeffM1 = CoeffM1(Inliers1); |
---|
| 189 | CoeffM2 = CoeffM2(Inliers2); |
---|
| 190 | |
---|
| 191 | % Check if the Matches are not mutual discard the one with less Coeff(Cross-Corrolation value) =============== |
---|
| 192 | Matches = [ Matches1 [Matches2(3:4,:); Matches2(1:2,:)]]; |
---|
| 193 | CoeffM = [ CoeffM1 CoeffM2]; |
---|
| 194 | % Min used different algorithm than SurFeature Matches |
---|
| 195 | [Inliers] = CleanMatch(Matches, CoeffM); % choose the matches with higher Coeff is the matches is not mutual |
---|
| 196 | Matches = Matches(:,Inliers); |
---|
| 197 | CoeffM = CoeffM(:,Inliers); |
---|
| 198 | if defaultPara.Flag.FlagRefinementDisp |
---|
| 199 | figure; plotmatches(I1,I2,Matches(1:2,:), Matches(3:4,:),repmat(1:size(Matches,2),2,1), 'Stacking','v','Interactive', 3); |
---|
| 200 | end |
---|
| 201 | |
---|
| 202 | % use Coeff as threshould to filter out error matches |
---|
| 203 | Mask = CoeffM > defaultPara.CoeffMThre; |
---|
| 204 | [inlier, Residual] = EpipoPrune(defaultPara, Pair, Matches, ImgScale1); |
---|
| 205 | Mark = Mask & Residual < defaultPara.ResidualThre; |
---|
| 206 | f1 = Matches(1:2,Mark); |
---|
| 207 | f2 = Matches(3:4,Mark); |
---|
| 208 | matches = repmat( 1:sum(Mark),2,1); |
---|
| 209 | end |
---|
| 210 | % ====================================================================================== |
---|
| 211 | |
---|
| 212 | if displayFlag |
---|
| 213 | figure(201); plotmatches(I1,I2,f1, f2,matches, 'Stacking','v','Interactive', 3); |
---|
| 214 | saveas(201,[ defaultPara.ScratchFolder Img1 '_' Img2 '_OccluMatchesPrune'],'jpg'); |
---|
| 215 | end |
---|
| 216 | end |
---|
| 217 | |
---|
| 218 | % Triangulation |
---|
| 219 | if ~isempty(matches) |
---|
| 220 | tempf1 = f1(:,matches(1,:)); |
---|
| 221 | tempf2 = f2(:,matches(2,:)); |
---|
| 222 | x_calib = [ inv(defaultPara.InrinsicK1)*[ tempf1; ones(1,size(tempf1,2))];... |
---|
| 223 | inv(defaultPara.InrinsicK2)*[ tempf2; ones(1,size(tempf2,2))]]; |
---|
| 224 | [ lamda1 lamda2 Error] = triangulation( defaultPara, Pair.R, Pair.T, x_calib); |
---|
| 225 | % notice lamda re-scale to local model scale |
---|
| 226 | lamda1 = lamda1./GlobalScale(1); |
---|
| 227 | lamda2 = lamda2./GlobalScale(2); |
---|
| 228 | % Storage the match result for later ReInference |
---|
| 229 | AddMatch2Model(defaultPara, Wrlname, lamda1, f1(:,matches(1,:)), ImgInfo1, ImgScale1, Img1Index, Img2Index, PostFixStrAfter, Error); |
---|
| 230 | AddMatch2Model(defaultPara, Wrlname, lamda2, f2(:,matches(2,:)), ImgInfo2, ImgScale2, Img2Index, Img1Index, PostFixStrAfter, Error); |
---|
| 231 | end |
---|
| 232 | |
---|
| 233 | % Storage the New Matches |
---|
| 234 | if defaultPara.Flag.FlagRefinementDisp |
---|
| 235 | disp('Storaging Occlusion Surf Features Matches'); |
---|
| 236 | end |
---|
| 237 | |
---|
| 238 | save([defaultPara.Fdir '/data/' Img1 '_' Img2 '_' PostFixStrAfter '.mat'],'f1','f2','matches', 'fail'); |
---|
| 239 | |
---|
| 240 | return; |
---|