[37] | 1 | % * This code was used in the following articles:
|
---|
| 2 | % * [1] Learning 3-D Scene Structure from a Single Still Image,
|
---|
| 3 | % * Ashutosh Saxena, Min Sun, Andrew Y. Ng,
|
---|
| 4 | % * In ICCV workshop on 3D Representation for Recognition (3dRR-07), 2007.
|
---|
| 5 | % * (best paper)
|
---|
| 6 | % * [2] 3-D Reconstruction from Sparse Views using Monocular Vision,
|
---|
| 7 | % * Ashutosh Saxena, Min Sun, Andrew Y. Ng,
|
---|
| 8 | % * In ICCV workshop on Virtual Representations and Modeling
|
---|
| 9 | % * of Large-scale environments (VRML), 2007.
|
---|
| 10 | % * [3] 3-D Depth Reconstruction from a Single Still Image,
|
---|
| 11 | % * Ashutosh Saxena, Sung H. Chung, Andrew Y. Ng.
|
---|
| 12 | % * International Journal of Computer Vision (IJCV), Aug 2007.
|
---|
| 13 | % * [6] Learning Depth from Single Monocular Images,
|
---|
| 14 | % * Ashutosh Saxena, Sung H. Chung, Andrew Y. Ng.
|
---|
| 15 | % * In Neural Information Processing Systems (NIPS) 18, 2005.
|
---|
| 16 | % *
|
---|
| 17 | % * These articles are available at:
|
---|
| 18 | % * http://make3d.stanford.edu/publications
|
---|
| 19 | % *
|
---|
| 20 | % * We request that you cite the papers [1], [3] and [6] in any of
|
---|
| 21 | % * your reports that uses this code.
|
---|
| 22 | % * Further, if you use the code in image3dstiching/ (multiple image version),
|
---|
| 23 | % * then please cite [2].
|
---|
| 24 | % *
|
---|
| 25 | % * If you use the code in third_party/, then PLEASE CITE and follow the
|
---|
| 26 | % * LICENSE OF THE CORRESPONDING THIRD PARTY CODE.
|
---|
| 27 | % *
|
---|
| 28 | % * Finally, this code is for non-commercial use only. For further
|
---|
| 29 | % * information and to obtain a copy of the license, see
|
---|
| 30 | % *
|
---|
| 31 | % * http://make3d.stanford.edu/publications/code
|
---|
| 32 | % *
|
---|
| 33 | % * Also, the software distributed under the License is distributed on an
|
---|
| 34 | % * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either
|
---|
| 35 | % * express or implied. See the License for the specific language governing
|
---|
| 36 | % * permissions and limitations under the License.
|
---|
| 37 | % *
|
---|
| 38 | % */
|
---|
| 39 | function []= MainProcess(defaultPara, Fdir); |
---|
| 40 | |
---|
| 41 | % This function is the main function tha handle: |
---|
| 42 | % 1) Extracting info mation : Exif GPS IMU |
---|
| 43 | % 2) MonoDepth estimation |
---|
| 44 | % 3) Ransac |
---|
| 45 | % 4) MetricReconstruction |
---|
| 46 | % 5) Re-inference |
---|
| 47 | % 6) Rendering |
---|
| 48 | |
---|
| 49 | % initialize parameters |
---|
| 50 | NotAllSurfCompute = 0; |
---|
| 51 | Type = '_RConS'; |
---|
| 52 | FlagInitialPathLearningCode = 0; |
---|
| 53 | FlagRenderMonoWrlJintly = 1; |
---|
| 54 | ops = sdpsettings('solver','sedumi','verbose',1); |
---|
| 55 | % parameter for Mono depth Estimation ------------------------ |
---|
| 56 | ScratchFolder = '/afs/cs/group/reconstruction3d/scratch/temp'; |
---|
| 57 | OutPutFolder = '/afs/cs/group/reconstruction3d/scratch/3DmodelMultipleImage/'; |
---|
| 58 | ParaFolder = '/afs/cs/group/reconstruction3d/scratch/Para/'; |
---|
| 59 | taskName = ''; |
---|
| 60 | Flag.DisplayFlag = 0; |
---|
| 61 | Flag.IntermediateStorage = 0; |
---|
| 62 | Flag.FeaturesOnly = 0; |
---|
| 63 | Flag.NormalizeFlag = 1; |
---|
| 64 | Flag.BeforeInferenceStorage = 0; |
---|
| 65 | Flag.NonInference = 0; |
---|
| 66 | Flag.AfterInferenceStorage = 1; |
---|
| 67 | % ------------------------------------------------------------ |
---|
| 68 | |
---|
| 69 | % initialize variables |
---|
| 70 | ImgInfo = []; |
---|
| 71 | ImgAddedList = []; |
---|
| 72 | |
---|
| 73 | % 1) Extracting info |
---|
| 74 | if NotAllSurfCompute |
---|
| 75 | system(['./surf/surfFeatureDir.sh ' Fdir]); |
---|
| 76 | end |
---|
| 77 | |
---|
| 78 | [ImgInfo] = ExifExtractDir(Fdir,ImgInfo); % Exif info |
---|
| 79 | [ImgInfo] = ExtractGPSInfo(defaultPara, Fdir, ImgInfo); % GPS info |
---|
| 80 | [ImgInfo] = ExtractRotationInfo( defaultPara, Fdir, ImgInfo); % IMU info |
---|
| 81 | |
---|
| 82 | % start loop by getting input |
---|
| 83 | while length(ImgAddedList) <= length(ImgInfo) |
---|
| 84 | |
---|
| 85 | % request user input |
---|
| 86 | NewImg = input('add image', 's'); |
---|
| 87 | |
---|
| 88 | % 2) MonoDepth estimation for New Imag------------------ |
---|
| 89 | NewImgPath = [Fdir '/jpg/' NewImg '.jpg']; |
---|
| 90 | if system(['ls ' ScratchFolder '/' NewImg '__AInfnew.mat']); |
---|
| 91 | cd ../LearningCode |
---|
| 92 | if ~FlagInitialPathLearningCode |
---|
| 93 | InitialPath; |
---|
| 94 | FlagInitialPathLearningCode = 1; |
---|
| 95 | end |
---|
| 96 | OneShot3dEfficient(NewImgPath, OutPutFolder,... |
---|
| 97 | taskName,...% taskname will append to the imagename and form the outputname |
---|
| 98 | ScratchFolder,... % ScratchFolder |
---|
| 99 | ParaFolder,... |
---|
| 100 | Flag... % All Flags 1) intermediate storage flag |
---|
| 101 | ); |
---|
| 102 | cd ../multipleImages |
---|
| 103 | end |
---|
| 104 | load( [ScratchFolder '/' NewImg '__AInfnew.mat']); |
---|
| 105 | |
---|
| 106 | % rename and clear vriables |
---|
| 107 | Depth2 = FitDepth; |
---|
| 108 | Sup2 = Sup; |
---|
| 109 | clear MedSup MultiScaleSupTable Ray RayOri SupNeighborTable SupOri depthMap maskG maskSky Sup FitDepth; |
---|
| 110 | % ------------------------------------------------------ |
---|
| 111 | |
---|
| 112 | % skip the following step if ImgAddedList is empty |
---|
| 113 | if isempty(ImgAddedList) |
---|
| 114 | Depth = Depth2; |
---|
| 115 | Sup = Sup2; |
---|
| 116 | save([Fdir '/data/' NewImg '_Data.mat'],'Depth','Sup'); |
---|
| 117 | ImgAddedList = [ImgAddedList NewImg]; |
---|
| 118 | continue; |
---|
| 119 | end |
---|
| 120 | |
---|
| 121 | % Use track concept to define track |
---|
| 122 | PairImg = input('Pair image in addedList', 's'); |
---|
| 123 | |
---|
| 124 | % build up data need for processing |
---|
| 125 | BuildDataPairImg; |
---|
| 126 | |
---|
| 127 | % 3) Ransac -------------------------------------------- |
---|
| 128 | % Estimate Rotation and translation |
---|
| 129 | [R T] = InitPoseMeas(defaultPara, ImgInfo1, ImgInfo2); |
---|
| 130 | T([2 5]) = 0; |
---|
| 131 | |
---|
| 132 | % Generate Constrain for SurfMatch searhing region |
---|
| 133 | [ Rc1, Rc2, ConS1, ConS2]=CalMatchSearchRegin(defaultPara, R, T, I1, I2, f1, f2, D1, D2, 1, 1); |
---|
| 134 | Vector2Ipoint([Rc1; ConS1],[Fdir '/surf/'],['RConS_' PairImg]); |
---|
| 135 | Vector2Ipoint([Rc2; ConS2],[Fdir '/surf/'],['RConS_' NewImg]); |
---|
| 136 | |
---|
| 137 | % Constrasin Search Surf Match |
---|
| 138 | % if system(['ls ' Fdir '/surf_matches/' PairImg '-' NewImg '.match_RConS']) ... |
---|
| 139 | % && system(['ls ' Fdir '/surf_matches/' NewImg '-' PairImg '.match_RConS']) |
---|
| 140 | cd match |
---|
| 141 | system(['./surfMatchRConS.sh ' Fdir ' ' PairImg ' ' NewImg]); |
---|
| 142 | cd .. |
---|
| 143 | % end |
---|
| 144 | % general Ransac |
---|
| 145 | [f1, f2, matches] = readSurfMatches(PairImg, NewImg, Fdir, Type, 0, 1); |
---|
| 146 | % displaySurfMatches(Fdir, PairImg, NewImg, Type, 0, 1); |
---|
| 147 | |
---|
| 148 | [DV DH] = size(Depth2); |
---|
| 149 | [IV IH] = size(I2); |
---|
| 150 | Scale = [[DV DH];[IV IH]]; |
---|
| 151 | [IND2]=ReScaleInd2Sub(Scale,f2(:,matches(2,:))); |
---|
| 152 | D2 = Depth2(IND2); |
---|
| 153 | [DV DH] = size(Depth1); |
---|
| 154 | [IV IH] = size(I1); |
---|
| 155 | Scale = [[DV DH];[IV IH]]; |
---|
| 156 | [IND1]=ReScaleInd2Sub(Scale,f1(:,matches(1,:))); |
---|
| 157 | D1 = Depth1(IND1); |
---|
| 158 | [F, inliers, NewDist, fail]=GeneralRansac(defaultPara, f1, f2, matches, D1, D2); |
---|
| 159 | figure; |
---|
| 160 | plotmatches(I1,I2,f1, f2,matches(:,inliers), 'Stacking', 'v', 'Interactive', 2); |
---|
| 161 | |
---|
| 162 | % maually remove outliers and re-estimate F |
---|
| 163 | matches = matches(:,inliers); |
---|
| 164 | [F, inliers, fail] = ransacmatches(defaultPara, f1, f2, matches, I1, I2, 1); |
---|
| 165 | |
---|
| 166 | % Nonlinear Refinement |
---|
| 167 | % [F, newinliers, fail] = RansacNonlinearReFinement(f1, f2, matches, F, inliers, I1, I2, 1); |
---|
| 168 | % ------------------------------------------------------ |
---|
| 169 | |
---|
| 170 | % 4) MetricReconstruction ------------------------------ |
---|
| 171 | % Initial depth estimated from fundamatal matrix |
---|
| 172 | E = defaultPara.InrinsicK2*F*defaultPara.InrinsicK1; |
---|
| 173 | x = [ inv(defaultPara.InrinsicK1)*[ f1(:,matches(1,inliers)); ones(1,length(inliers))];... |
---|
| 174 | inv(defaultPara.InrinsicK2)*[ f2(:,matches(2,inliers)); ones(1,length(inliers))]]; |
---|
| 175 | [ R_f, T_f, lamda1, lamda2, inlier] = EstPose( E, x, R(1:3,:), T(1:3), false); |
---|
| 176 | |
---|
| 177 | % Normalise each set of points so that the origin is at centroid and |
---|
| 178 | % mean distance from origin is sqrt(2). normalise2dpts also ensures the |
---|
| 179 | % scale parameter is 1. Note that 'fundmatrix' will also call |
---|
| 180 | % 'normalise2dpts' but the code in 'ransac' that calls the distance |
---|
| 181 | % function will not - so it is best that we normalise beforehand. |
---|
| 182 | x1 = defaultPara.InrinsicK1*x(1:3,inlier); |
---|
| 183 | x2 = defaultPara.InrinsicK2*x(4:6,inlier); |
---|
| 184 | [x1_normalized, T1] = normalise2dpts(x1); |
---|
| 185 | [x2_normalized, T2] = normalise2dpts(x2); |
---|
| 186 | |
---|
| 187 | % Projective resonstruction |
---|
| 188 | [ P, X, Outliner] = ProjectionFactorization( 1, 1, ... |
---|
| 189 | cat(3, x1_normalized, x2_normalized), cat(3,lamda1(inlier)', lamda2(inlier)')); |
---|
| 190 | % [ P, X, Outliner] = ProjectionFactorization_missing_data... |
---|
| 191 | % ( ARes, BRes, x_im, inc, depth); |
---|
| 192 | |
---|
| 193 | % Find out the H (projective transformation) |
---|
| 194 | matches = matches(:,inliers); |
---|
| 195 | x1 = [f1(:,matches(1,inlier)); ones(1,length(inlier))]; |
---|
| 196 | x2 = [f2(:,matches(1,inlier)); ones(1,length(inlier))]; |
---|
| 197 | Scale = [[DV DH];[IV IH]]; |
---|
| 198 | [IND1]=ReScaleInd2Sub(Scale,f1(:,matches(1,inlier))); |
---|
| 199 | D1 = Depth1(IND1); |
---|
| 200 | Scale = [[DV DH];[IV IH]]; |
---|
| 201 | [IND2]=ReScaleInd2Sub(Scale,f2(:,matches(1,inlier))); |
---|
| 202 | D2 = Depth2(IND2); |
---|
| 203 | [R_mr T_mr D1_modified D2_modified] = MetricReconEstPose(P, R(1:3,:), T(1:3), T1, T2, D1, D2, x1, x2, defaultPara); |
---|
| 204 | |
---|
| 205 | % Triangulate the depth |
---|
| 206 | if false |
---|
| 207 | x1 = inv(T1)*x1; |
---|
| 208 | x2 = inv(T1)*x2; |
---|
| 209 | x1(:,Outliner) = []; |
---|
| 210 | x2(:,Outliner) = []; |
---|
| 211 | X = [x1; x2]; |
---|
| 212 | [lamda1, lamda2] = Triangulate( R_mr, T_mr, X); |
---|
| 213 | % Rescale the Depth1 and Depth2 |
---|
| 214 | x1 = defaultPara.InrinsicK1*x1; |
---|
| 215 | x2 = defaultPara.InrinsicK2*x2; |
---|
| 216 | [DV DH] = size(Depth1); |
---|
| 217 | [IV IH] = size(I1); |
---|
| 218 | Scale = [[DV DH];[IV IH]]; |
---|
| 219 | [IND1]=ReScaleInd2Sub(Scale,x1); |
---|
| 220 | D1 = Depth1(IND1); |
---|
| 221 | [DV DH] = size(Depth2); |
---|
| 222 | [IV IH] = size(I2); |
---|
| 223 | Scale = [[DV DH];[IV IH]]; |
---|
| 224 | [IND2]=ReScaleInd2Sub(Scale,x2); |
---|
| 225 | D2 = Depth2(IND2); |
---|
| 226 | Scale1 = sdpvar(1,1); |
---|
| 227 | Scale2 = sdpvar(1,1); |
---|
| 228 | F = set(Scale1>=0)+set(Scale2 >=0); |
---|
| 229 | sol = solvesdp(F,norm(lamda1 - Scale1*D1', 1)+ norm(lamda2 - Scale2*D2', 1),ops); |
---|
| 230 | Scale1 = double(Scale1); |
---|
| 231 | Scale2 = double(Scale2); |
---|
| 232 | Depth1 = Depth1*Scale1; |
---|
| 233 | Depth2 = Depth2*Scale2; |
---|
| 234 | else |
---|
| 235 | Scale1 = median(D1_modified./D1); |
---|
| 236 | Scale2 = median(D2_modified./D2); |
---|
| 237 | Depth1 = Depth1*Scale1; |
---|
| 238 | Depth2 = Depth2*Scale2; |
---|
| 239 | % Depth1(IND1) = D1_modified; |
---|
| 240 | % Depth2(IND2) = D2_modified; |
---|
| 241 | end |
---|
| 242 | |
---|
| 243 | % ------------------------------------------------------ |
---|
| 244 | |
---|
| 245 | % Test R_mr and T_mr to show the Mono-VRml jointly according to R_mr and T_mr |
---|
| 246 | if FlagRenderMonoWrlJintly |
---|
| 247 | [WrlPosition1 PositionTex1] = PreWrlData(defaultPara.InrinsicK1, Depth1, I1, eye(3), zeros(3,1)); |
---|
| 248 | WrlFacestHroiReduce(WrlPosition1, PositionTex1, Sup1, PairImg, [PairImg '_' NewImg '_MonoJointly'], OutPutFolder, 0, 0); |
---|
| 249 | system(['cp ' Fdir '/jpg/' PairImg '.jpg ' OutPutFolder PairImg '.jpg']); |
---|
| 250 | [WrlPosition2 PositionTex2] = PreWrlData(defaultPara.InrinsicK2, Depth2, I2, R_mr, T_mr); |
---|
| 251 | WrlFacestHroiReduce(WrlPosition2, PositionTex2, Sup2, NewImg, [PairImg '_' NewImg '_MonoJointly'], OutPutFolder, 0, 1); |
---|
| 252 | system(['cp ' Fdir '/jpg/' NewImg '.jpg ' OutPutFolder NewImg '.jpg']); |
---|
| 253 | end |
---|
| 254 | |
---|
| 255 | % 5) Reinfererce |
---|
| 256 | |
---|
| 257 | ImgAddedList = [ImgAddedList NewImg]; |
---|
| 258 | |
---|
| 259 | |
---|
| 260 | % save Data |
---|
| 261 | Depth = Depth2; |
---|
| 262 | Sup = Sup2; |
---|
| 263 | save([Fdir '/data/' NewImg '_Data.mat'],'Depth','Sup'); |
---|
| 264 | end |
---|