[37] | 1 | % * This code was used in the following articles:
|
---|
| 2 | % * [1] Learning 3-D Scene Structure from a Single Still Image,
|
---|
| 3 | % * Ashutosh Saxena, Min Sun, Andrew Y. Ng,
|
---|
| 4 | % * In ICCV workshop on 3D Representation for Recognition (3dRR-07), 2007.
|
---|
| 5 | % * (best paper)
|
---|
| 6 | % * [2] 3-D Reconstruction from Sparse Views using Monocular Vision,
|
---|
| 7 | % * Ashutosh Saxena, Min Sun, Andrew Y. Ng,
|
---|
| 8 | % * In ICCV workshop on Virtual Representations and Modeling
|
---|
| 9 | % * of Large-scale environments (VRML), 2007.
|
---|
| 10 | % * [3] 3-D Depth Reconstruction from a Single Still Image,
|
---|
| 11 | % * Ashutosh Saxena, Sung H. Chung, Andrew Y. Ng.
|
---|
| 12 | % * International Journal of Computer Vision (IJCV), Aug 2007.
|
---|
| 13 | % * [6] Learning Depth from Single Monocular Images,
|
---|
| 14 | % * Ashutosh Saxena, Sung H. Chung, Andrew Y. Ng.
|
---|
| 15 | % * In Neural Information Processing Systems (NIPS) 18, 2005.
|
---|
| 16 | % *
|
---|
| 17 | % * These articles are available at:
|
---|
| 18 | % * http://make3d.stanford.edu/publications
|
---|
| 19 | % *
|
---|
| 20 | % * We request that you cite the papers [1], [3] and [6] in any of
|
---|
| 21 | % * your reports that uses this code.
|
---|
| 22 | % * Further, if you use the code in image3dstiching/ (multiple image version),
|
---|
| 23 | % * then please cite [2].
|
---|
| 24 | % *
|
---|
| 25 | % * If you use the code in third_party/, then PLEASE CITE and follow the
|
---|
| 26 | % * LICENSE OF THE CORRESPONDING THIRD PARTY CODE.
|
---|
| 27 | % *
|
---|
| 28 | % * Finally, this code is for non-commercial use only. For further
|
---|
| 29 | % * information and to obtain a copy of the license, see
|
---|
| 30 | % *
|
---|
| 31 | % * http://make3d.stanford.edu/publications/code
|
---|
| 32 | % *
|
---|
| 33 | % * Also, the software distributed under the License is distributed on an
|
---|
| 34 | % * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either
|
---|
| 35 | % * express or implied. See the License for the specific language governing
|
---|
| 36 | % * permissions and limitations under the License.
|
---|
| 37 | % *
|
---|
| 38 | % */
|
---|
| 39 | function [defaultPara ImgInfo] = PairReInferenceSepRender(defaultPara, Pair, ImgInfo, FlagFirstPair) |
---|
| 40 | |
---|
| 41 | % This function is the meta function of 2 decompose inference |
---|
| 42 | % Input: |
---|
| 43 | % defaultPara - camera intrinsic parameters |
---|
| 44 | % Pair - Pair image info- matches camera extrinsic |
---|
| 45 | % ImgInfo - Sup, Ray, MultiScaleSup, SupNeighbor, Depth, Constrain (previous) |
---|
| 46 | % Return: |
---|
| 47 | % ImgInfo - add new Constrain |
---|
| 48 | % initialize parameter |
---|
| 49 | NegI = diag([1 1 -1]); |
---|
| 50 | NumMatches = length(Pair.Xim); |
---|
| 51 | Img1 = ImgInfo(1).ExifInfo.IDName; |
---|
| 52 | Img2 = ImgInfo(2).ExifInfo.IDName; |
---|
| 53 | I1=imreadbw([defaultPara.Fdir '/pgm/' Img1 '.pgm']); % function from sift |
---|
| 54 | I2=imreadbw([defaultPara.Fdir '/pgm/' Img2 '.pgm']); % function from sift |
---|
| 55 | |
---|
| 56 | % ========================================================== |
---|
| 57 | % ConStrain generation |
---|
| 58 | % 1) Triangulated Depth |
---|
| 59 | x_calib = [ inv(defaultPara.InrinsicK1)*[ Pair.Xim(1:2,:); ones(1, NumMatches)];... |
---|
| 60 | inv(defaultPara.InrinsicK2)*[ Pair.Xim(3:4,:); ones(1, NumMatches)]]; |
---|
| 61 | TriDepth1 = Pair.lamda(1,:); |
---|
| 62 | TriDepth2 = Pair.lamda(2,:); |
---|
| 63 | |
---|
| 64 | % ================= important check scaling stanford =============== |
---|
| 65 | ScaleImg = size(I1); |
---|
| 66 | ScaleDepth = size(ImgInfo(1).Model.Depth.FitDepth); |
---|
| 67 | [IND1] = ProjPosi2Mask( ScaleImg, ScaleDepth, Pair.Xim([1 2],:)); |
---|
| 68 | %StichedDeph1 = ImgInfo(1).Model.Depth.RawDepth(IND1); |
---|
| 69 | StichedDeph1 = ImgInfo(1).Model.Depth.FitDepth(IND1);%Min used FitDepth July 1st |
---|
| 70 | if mean(abs(StichedDeph1 - TriDepth1)) < mean( abs( StichedDeph1*Pair.DepthScale(1) - TriDepth1)) |
---|
| 71 | ImgInfo(1).Model.Depth.FitDepth = ImgInfo(1).Model.Depth.FitDepth/Pair.DepthScale(1); |
---|
| 72 | ImgInfo(1).Model.Depth.RawDepth = ImgInfo(1).Model.Depth.RawDepth/Pair.DepthScale(1); |
---|
| 73 | disp('Wrong Scaling'); |
---|
| 74 | end |
---|
| 75 | |
---|
| 76 | ScaleImg = size(I2); |
---|
| 77 | ScaleDepth = size(ImgInfo(2).Model.Depth.FitDepth); |
---|
| 78 | [IND2] = ProjPosi2Mask( ScaleImg, ScaleDepth, Pair.Xim([3 4],:)); |
---|
| 79 | %StichedDeph2 = ImgInfo(2).Model.Depth.RawDepth(IND2); |
---|
| 80 | StichedDeph2 = ImgInfo(2).Model.Depth.FitDepth(IND2);%Min used FitDepth July 1st |
---|
| 81 | if mean(abs(StichedDeph2 - TriDepth2)) < mean( abs( StichedDeph2*Pair.DepthScale(2) - TriDepth2)) |
---|
| 82 | ImgInfo(2).Model.Depth.FitDepth = ImgInfo(2).Model.Depth.FitDepth/Pair.DepthScale(2); |
---|
| 83 | ImgInfo(2).Model.Depth.RawDepth = ImgInfo(2).Model.Depth.RawDepth/Pair.DepthScale(2); |
---|
| 84 | disp('Wrong Scaling'); |
---|
| 85 | end |
---|
| 86 | % ================================================================== |
---|
| 87 | RayNormfactor1 = sqrt( sum(x_calib(1:3,:).^2,1)); |
---|
| 88 | RayNormfactor2 = sqrt( sum(x_calib(4:6,:).^2,1)); |
---|
| 89 | if FlagFirstPair |
---|
| 90 | % 2) Ground orientaion and level |
---|
| 91 | [GroundLevel] = CalGroundLevel(defaultPara, ImgInfo, Pair); |
---|
| 92 | else |
---|
| 93 | GroundLevel = LoadModelStatus(defaultPara.Fdir, defaultPara.Wrlname, Img1, 'GroundLevel'); |
---|
| 94 | end |
---|
| 95 | |
---|
| 96 | [D1 IND1] = PorjPosi2Depth(size(I1), size(ImgInfo(1).Model.Depth.FitDepth), Pair.Xim(1:2,:), ImgInfo(1).Model.Depth.FitDepth); % Need IND1 |
---|
| 97 | [D2 IND2] = PorjPosi2Depth(size(I2), size(ImgInfo(2).Model.Depth.FitDepth), Pair.Xim(3:4,:), ImgInfo(1).Model.Depth.FitDepth); % Need IND2 |
---|
| 98 | clear I1 I2; |
---|
| 99 | |
---|
| 100 | % ==== General Parameters =============== |
---|
| 101 | Default.RenderFlag = defaultPara.RenderFlag; |
---|
| 102 | [Default.VertYNuDepth Default.HoriXNuDepth] = size(ImgInfo(1).Model.Depth.FitDepth); |
---|
| 103 | Default.fy = 2400.2091651084; |
---|
| 104 | Default.fx = 2407.3312729885838; |
---|
| 105 | Default.Ox = 1110.7122391785729;%2272/2; % |
---|
| 106 | Default.Oy = 833.72104535435108;%1704/2; % |
---|
| 107 | Default.a_default = 2272/Default.fx; %0.70783777; %0.129; % horizontal physical size of image plane normalized to focal length (in meter) |
---|
| 108 | Default.b_default = 1704/Default.fy; %0.946584169;%0.085; % vertical physical size of image plane normalized to focal length (in meter) |
---|
| 109 | Default.Ox_default = 1-Default.Ox/2272;%0.489272914; % camera origin offset from the image center in horizontal direction |
---|
| 110 | Default.Oy_default = 1-Default.Oy/1704;%0.488886982; % camera origin offset from the image center in vertical direction |
---|
| 111 | Default.MinTriEffectPercent = 5; % 5 % higher the CoPlaner term |
---|
| 112 | Default.FarestTriDist = 20;%0.1; % unit in pixel; 1 still too samll |
---|
| 113 | %Default.TriCountSupThre = 15; |
---|
| 114 | Default.TriCountSupThre = size(ImgInfo(1).Model.Constrain.RayMatche,1)*0.1; |
---|
| 115 | % ========================================== First Model |
---|
| 116 | % highly changable |
---|
| 117 | Default.OutPutFolder = [ defaultPara.OutPutFolder Img1 '/']; |
---|
| 118 | Default.ScratchFolder = Default.OutPutFolder; |
---|
| 119 | Default.Wrlname{1} = [defaultPara.Wrlname '_' Img1]; |
---|
| 120 | Default.Flag.AfterInferenceStorage = 0; |
---|
| 121 | |
---|
| 122 | AappendOpt = 0; % not append since generate seperate .wrl files |
---|
| 123 | ASupMatched = ImgInfo(1).Model.Sup(IND1)'; |
---|
| 124 | mask = ASupMatched == 0; |
---|
| 125 | ASupMatched(mask)=[]; |
---|
| 126 | ARayMatched = (x_calib(1:3,:)./(repmat( RayNormfactor1,3,1)))'; |
---|
| 127 | ARayMatched(mask,:) = []; |
---|
| 128 | ADepth_modified = TriDepth1.*RayNormfactor1/Pair.DepthScale(1); % /Pair.DepthScale(1) bing to Ori_scale |
---|
| 129 | ADepth_modified(:,mask) = []; |
---|
| 130 | Aconstrain.RayMatched = ImgInfo(1).Model.Constrain.RayMatche; |
---|
| 131 | Aconstrain.Depth_modified = ImgInfo(1).Model.Constrain.Depth_modified; |
---|
| 132 | Aconstrain.SupMatched = ImgInfo(1).Model.Constrain.SupMatched; |
---|
| 133 | ASup = ImgInfo(1).Model.Sup; |
---|
| 134 | ASupOri = ASup; |
---|
| 135 | AdepthMap = ImgInfo(1).Model.Depth.RawDepth; % or RawDepth |
---|
| 136 | ARayOri = ImgInfo(1).Model.Ray; |
---|
| 137 | ARayAll = ARayOri; |
---|
| 138 | ASupNeighborTable = ImgInfo(1).Model.SupNeighborTable; |
---|
| 139 | AmaskSky = ImgInfo(1).Model.maskSky; |
---|
| 140 | AmaskG = ImgInfo(1).Model.maskG; |
---|
| 141 | AMultiScaleSupTable = ImgInfo(1).Model.MultiScaleSupTable; |
---|
| 142 | |
---|
| 143 | Default.filename{1} = [ strrep(ImgInfo(1).ExifInfo.name, '.jpg','') '_']; % Min Modified Aug 18th |
---|
| 144 | % Default.filename{1} = ['../' strrep(ImgInfo(1).ExifInfo.name, '.jpg','')]; |
---|
| 145 | % =====================Meta Model Building ================= |
---|
| 146 | if FlagFirstPair |
---|
| 147 | Rotation = defaultPara.R; |
---|
| 148 | Translation = defaultPara.T; |
---|
| 149 | UniScale = defaultPara.Scale; |
---|
| 150 | else |
---|
| 151 | Rotation = LoadModelStatus( defaultPara.Fdir, defaultPara.Wrlname, Img1, 'R'); |
---|
| 152 | Translation = LoadModelStatus( defaultPara.Fdir, defaultPara.Wrlname, Img1, 'T'); |
---|
| 153 | UniScale = LoadModelStatus(defaultPara.Fdir, defaultPara.Wrlname, Img1, 'Scale'); |
---|
| 154 | end |
---|
| 155 | |
---|
| 156 | % first Img1 |
---|
| 157 | Path = [ defaultPara.OutPutFolder defaultPara.Wrlname '.wrl']; |
---|
| 158 | InLinePath = ['./' Img1 '/' Default.Wrlname{1} '.wrl']; |
---|
| 159 | R = Rotation; |
---|
| 160 | % Possibly to be wrong ========== |
---|
| 161 | % Angle = recoverAlphasFromU(reshape(R,1,[])); |
---|
| 162 | % Q = GetQauternionFrom2Rotation(zeros(3,1), Angle, false);%[1 0 0 0]'; |
---|
| 163 | Q = Rotation2Q(NegI*R*NegI);% for Wrl (-) z component; |
---|
| 164 | if any(isnan(Q)) |
---|
| 165 | Q = zeros(4,1); |
---|
| 166 | end |
---|
| 167 | % =============================== |
---|
| 168 | T = Translation; |
---|
| 169 | WRLT = T; |
---|
| 170 | WRLT(3) = -WRLT(3);% for Wrl (-) z component; |
---|
| 171 | Scale = UniScale; |
---|
| 172 | BuildVrmlMetaModel(FlagFirstPair, defaultPara.OutPutFolder, Path, InLinePath, Q, WRLT, repmat(Scale,3,1)); |
---|
| 173 | % ============================================================ |
---|
| 174 | if true |
---|
| 175 | [ ImgInfo(1).Model.PlaneParaInfo] = PlaneParaMRFTriangulateOneShot( Default, Rotation, Translation, AappendOpt, ... |
---|
| 176 | [ ARayMatched; Aconstrain.RayMatched],... |
---|
| 177 | [ ADepth_modified Aconstrain.Depth_modified]',... |
---|
| 178 | [ ASupMatched; Aconstrain.SupMatched],... |
---|
| 179 | [ ], [ ], [ ], [],... |
---|
| 180 | ASup, ASupOri, [], AdepthMap, zeros(size(AdepthMap)), ARayOri, ARayAll, ... |
---|
| 181 | ASupNeighborTable, [], AmaskSky, AmaskG,... |
---|
| 182 | 'cvx_allL1Norm',1,... |
---|
| 183 | [], [], AMultiScaleSupTable, [], [], [], false, Scale, Rotation, GroundLevel);% eye(3), GroundLevel might be wrong //Min check |
---|
| 184 | end |
---|
| 185 | |
---|
| 186 | % Important Storage the Triangulated info in local scale |
---|
| 187 | ImgInfo(1).Model.Constrain.RayMatche = [ ARayMatched; Aconstrain.RayMatched]; |
---|
| 188 | ImgInfo(1).Model.Constrain.Depth_modified = [ ADepth_modified Aconstrain.Depth_modified]; |
---|
| 189 | ImgInfo(1).Model.Constrain.SupMatched = [ ASupMatched; Aconstrain.SupMatched]; |
---|
| 190 | model = ImgInfo(1).Model; |
---|
| 191 | ImgName = strrep(ImgInfo(1).ExifInfo.name,'.jpg',''); |
---|
| 192 | save( [defaultPara.ScratchFolder ImgName '/' defaultPara.Wrlname '_' ImgName '_NonMono.mat'], 'model'); % add prefix defaultPara.Wrlname to distinguish model |
---|
| 193 | % =================================== Second Model |
---|
| 194 | Default.OutPutFolder = [ defaultPara.OutPutFolder Img2 '/']; |
---|
| 195 | Default.ScratchFolder = Default.OutPutFolder; |
---|
| 196 | Default.Wrlname{1} = [defaultPara.Wrlname '_' Img2]; |
---|
| 197 | Default.Flag.AfterInferenceStorage = 0; |
---|
| 198 | |
---|
| 199 | Default.RenderFlag = 1;%defaultPara.LastImgFlag;//Min changed to render anyway |
---|
| 200 | AappendOpt = 0; % not append since generate seperate .wrl files |
---|
| 201 | ASupMatched = ImgInfo(2).Model.Sup(IND2)'; |
---|
| 202 | mask = ASupMatched == 0; |
---|
| 203 | ASupMatched(mask)=[]; |
---|
| 204 | ARayMatched = (x_calib(4:6,:)./(repmat( RayNormfactor2,3,1)))'; |
---|
| 205 | ARayMatched(mask,:) = []; |
---|
| 206 | ADepth_modified = TriDepth2.*RayNormfactor2/Pair.DepthScale(2);% /Pair.DepthScale(2) Bing to Ori_Scale |
---|
| 207 | ADepth_modified(:,mask) = []; |
---|
| 208 | Aconstrain.RayMatched = ImgInfo(2).Model.Constrain.RayMatche; |
---|
| 209 | Aconstrain.Depth_modified = ImgInfo(2).Model.Constrain.Depth_modified; |
---|
| 210 | Aconstrain.SupMatched = ImgInfo(2).Model.Constrain.SupMatched; |
---|
| 211 | ASup = ImgInfo(2).Model.Sup; |
---|
| 212 | ASupOri = ASup; |
---|
| 213 | AdepthMap = ImgInfo(2).Model.Depth.RawDepth; % or RawDepth |
---|
| 214 | ARayOri = ImgInfo(2).Model.Ray; |
---|
| 215 | ARayAll = ARayOri; |
---|
| 216 | ASupNeighborTable = ImgInfo(2).Model.SupNeighborTable; |
---|
| 217 | AmaskSky = ImgInfo(2).Model.maskSky; |
---|
| 218 | AmaskG = ImgInfo(2).Model.maskG; |
---|
| 219 | AMultiScaleSupTable = ImgInfo(2).Model.MultiScaleSupTable; |
---|
| 220 | |
---|
| 221 | Default.filename{1} = [ strrep(ImgInfo(2).ExifInfo.name, '.jpg','') '_']; % Min Modified Aug 18th |
---|
| 222 | % Default.filename{1} = ['../' strrep(ImgInfo(2).ExifInfo.name, '.jpg','')]; |
---|
| 223 | % ========Add model or modify model in Global Meta Model file================= |
---|
| 224 | % Second Img2 |
---|
| 225 | Path = [ defaultPara.OutPutFolder defaultPara.Wrlname '.wrl']; |
---|
| 226 | InLinePath = ['./' Img2 '/' Default.Wrlname{1} '.wrl']; |
---|
| 227 | R = Rotation*Pair.R'; |
---|
| 228 | Q = Rotation2Q(NegI*R*NegI); |
---|
| 229 | if any(isnan(Q)) |
---|
| 230 | Q = zeros(4,1); |
---|
| 231 | end |
---|
| 232 | % =============================== |
---|
| 233 | T = Translation + Rotation*(-Pair.R'*(Pair.T/Pair.DepthScale(1)*UniScale )); |
---|
| 234 | WRLT = T; |
---|
| 235 | WRLT(3) = -WRLT(3);% for Wrl (-) z component; |
---|
| 236 | Scale = Pair.DepthScale(2)/Pair.DepthScale(1)*UniScale; |
---|
| 237 | BuildVrmlMetaModel(0, defaultPara.OutPutFolder, Path, InLinePath, Q, WRLT, repmat(Scale,3,1)); |
---|
| 238 | % ============================================================================true |
---|
| 239 | if true;%Default.RenderFlag % Since using inline to link VRML render it anyway |
---|
| 240 | [ ImgInfo(2).Model.PlaneParaInfo] = PlaneParaMRFTriangulateOneShot( Default, R, T, AappendOpt, ... |
---|
| 241 | [ ARayMatched; Aconstrain.RayMatched],... |
---|
| 242 | [ ADepth_modified Aconstrain.Depth_modified]',... |
---|
| 243 | [ ASupMatched; Aconstrain.SupMatched],... |
---|
| 244 | [ ], [ ], [ ], [],... |
---|
| 245 | ASup, ASupOri, [], AdepthMap, zeros(size(AdepthMap)), ARayOri, ARayAll, ... |
---|
| 246 | ASupNeighborTable, [], AmaskSky, AmaskG,... |
---|
| 247 | 'cvx_allL1Norm',1,... |
---|
| 248 | [], [], AMultiScaleSupTable, [], [], [], false, Scale, R, GroundLevel); % eye(3), GroundLevel might be wrong //Min check |
---|
| 249 | end |
---|
| 250 | |
---|
| 251 | % Important Storage the Triangulated info in local scale |
---|
| 252 | ImgInfo(2).Model.Constrain.RayMatche = [ ARayMatched; Aconstrain.RayMatched]; |
---|
| 253 | ImgInfo(2).Model.Constrain.Depth_modified = [ ADepth_modified Aconstrain.Depth_modified]; |
---|
| 254 | ImgInfo(2).Model.Constrain.SupMatched = [ ASupMatched; Aconstrain.SupMatched]; |
---|
| 255 | model = ImgInfo(2).Model; |
---|
| 256 | ImgName = strrep(ImgInfo(2).ExifInfo.name,'.jpg',''); |
---|
| 257 | save( [defaultPara.ScratchFolder ImgName '/' defaultPara.Wrlname '_' ImgName '_NonMono.mat'], 'model');% add prefix defaultPara.Wrlname to distinguish model |
---|
| 258 | |
---|
| 259 | |
---|
| 260 | % Save GroundLevel, R, T, and, Scale info in /data/ModelStatus for adding |
---|
| 261 | % new image into the current model |
---|
| 262 | SaveModelSatus( defaultPara, defaultPara.Wrlname, {Img1,Img2}, GroundLevel, R, T, Scale, FlagFirstPair) |
---|
| 263 | |
---|
| 264 | return; |
---|
| 265 | |
---|