[37] | 1 | % * This code was used in the following articles:
|
---|
| 2 | % * [1] Learning 3-D Scene Structure from a Single Still Image,
|
---|
| 3 | % * Ashutosh Saxena, Min Sun, Andrew Y. Ng,
|
---|
| 4 | % * In ICCV workshop on 3D Representation for Recognition (3dRR-07), 2007.
|
---|
| 5 | % * (best paper)
|
---|
| 6 | % * [2] 3-D Reconstruction from Sparse Views using Monocular Vision,
|
---|
| 7 | % * Ashutosh Saxena, Min Sun, Andrew Y. Ng,
|
---|
| 8 | % * In ICCV workshop on Virtual Representations and Modeling
|
---|
| 9 | % * of Large-scale environments (VRML), 2007.
|
---|
| 10 | % * [3] 3-D Depth Reconstruction from a Single Still Image,
|
---|
| 11 | % * Ashutosh Saxena, Sung H. Chung, Andrew Y. Ng.
|
---|
| 12 | % * International Journal of Computer Vision (IJCV), Aug 2007.
|
---|
| 13 | % * [6] Learning Depth from Single Monocular Images,
|
---|
| 14 | % * Ashutosh Saxena, Sung H. Chung, Andrew Y. Ng.
|
---|
| 15 | % * In Neural Information Processing Systems (NIPS) 18, 2005.
|
---|
| 16 | % *
|
---|
| 17 | % * These articles are available at:
|
---|
| 18 | % * http://make3d.stanford.edu/publications
|
---|
| 19 | % *
|
---|
| 20 | % * We request that you cite the papers [1], [3] and [6] in any of
|
---|
| 21 | % * your reports that uses this code.
|
---|
| 22 | % * Further, if you use the code in image3dstiching/ (multiple image version),
|
---|
| 23 | % * then please cite [2].
|
---|
| 24 | % *
|
---|
| 25 | % * If you use the code in third_party/, then PLEASE CITE and follow the
|
---|
| 26 | % * LICENSE OF THE CORRESPONDING THIRD PARTY CODE.
|
---|
| 27 | % *
|
---|
| 28 | % * Finally, this code is for non-commercial use only. For further
|
---|
| 29 | % * information and to obtain a copy of the license, see
|
---|
| 30 | % *
|
---|
| 31 | % * http://make3d.stanford.edu/publications/code
|
---|
| 32 | % *
|
---|
| 33 | % * Also, the software distributed under the License is distributed on an
|
---|
| 34 | % * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either
|
---|
| 35 | % * express or implied. See the License for the specific language governing
|
---|
| 36 | % * permissions and limitations under the License.
|
---|
| 37 | % *
|
---|
| 38 | % */
|
---|
| 39 | function [PointPix1 PointDepth1 TargetIND1 POriReprojM1 FieldOccluPix1 OcluedDist1 OccluedFaceSetIND1 FieldFaceSetIDRemained1... |
---|
| 40 | PointPix2 PointDepth2 TargetIND2 POriReprojM2 FieldOccluPix2 OcluedDist2 OccluedFaceSetIND2 FieldFaceSetIDRemained2]=... |
---|
| 41 | FindOccluPair(defaultPara, ModelInfo1, ModelInfo2, Pair, GlobalScale, SurfFlag) |
---|
| 42 | |
---|
| 43 | % Function find the closed occlu point and output a region on epipolar line |
---|
| 44 | % to search for dense match |
---|
| 45 | % Hacking Fix Parameters ============================== |
---|
| 46 | Default.fy = 2400.2091651084; |
---|
| 47 | Default.fx = 2407.3312729885838; |
---|
| 48 | Default.Ox = 1110.7122391785729;%2272/2; % |
---|
| 49 | Default.Oy = 833.72104535435108;%1704/2; % |
---|
| 50 | Default.a_default = 2272/Default.fx; %0.70783777; %0.129; % horizontal physical size of image plane normalized to focal length (in meter) |
---|
| 51 | Default.b_default = 1704/Default.fy; %0.946584169;%0.085; % vertical physical size of image plane normalized to focal length (in meter) |
---|
| 52 | Default.Ox_default = 1-Default.Ox/2272;%0.489272914; % camera origin offset from the image center in horizontal direction |
---|
| 53 | Default.Oy_default = 1-Default.Oy/1704;%0.488886982; % camera origin offset from the image center in vertical direction |
---|
| 54 | [Default.VertYNuDepth Default.HoriXNuDepth] = size(ModelInfo1.Model.PlaneParaInfo.FitDepth); |
---|
| 55 | % ===================================================== |
---|
| 56 | |
---|
| 57 | % loading Image |
---|
| 58 | Img1 = ModelInfo1.ExifInfo.IDName; |
---|
| 59 | I1 = imreadbw([defaultPara.Fdir '/pgm/' Img1 '.pgm']); |
---|
| 60 | Img2 = ModelInfo2.ExifInfo.IDName; |
---|
| 61 | I2 = imreadbw([defaultPara.Fdir '/pgm/' Img2 '.pgm']); |
---|
| 62 | [Iy1 Ix1] = size(I1); |
---|
| 63 | [DepthY1 DepthX1] = size(ModelInfo1.Model.Depth.FitDepth); |
---|
| 64 | [Iy2 Ix2] = size(I2); |
---|
| 65 | [DepthY2 DepthX2] = size(ModelInfo2.Model.Depth.FitDepth); |
---|
| 66 | |
---|
| 67 | % Proper Scaling the Depth 3-d Position and PlaneParameters (all in global scale) |
---|
| 68 | ModelInfo1.Model.PlaneParaInfo.PlanePara = ModelInfo1.Model.PlaneParaInfo.PlanePara/GlobalScale(1); |
---|
| 69 | ModelInfo1.Model.PlaneParaInfo.FitDepth = ModelInfo1.Model.PlaneParaInfo.FitDepth*GlobalScale(1); |
---|
| 70 | ModelInfo1.Model.PlaneParaInfo.Position3DFited = permute( ModelInfo1.Model.PlaneParaInfo.Position3DFited*GlobalScale(1),[ 3 1 2]); |
---|
| 71 | ModelInfo2.Model.PlaneParaInfo.PlanePara = ModelInfo2.Model.PlaneParaInfo.PlanePara/GlobalScale(2); |
---|
| 72 | ModelInfo2.Model.PlaneParaInfo.FitDepth = ModelInfo2.Model.PlaneParaInfo.FitDepth*GlobalScale(1); |
---|
| 73 | ModelInfo2.Model.PlaneParaInfo.Position3DFited = permute( ModelInfo2.Model.PlaneParaInfo.Position3DFited*GlobalScale(2),[ 3 1 2]); |
---|
| 74 | |
---|
| 75 | % correct negative z positiom |
---|
| 76 | ModelInfo1.Model.PlaneParaInfo.Position3DFited(3,:) = -ModelInfo1.Model.PlaneParaInfo.Position3DFited(3,:); |
---|
| 77 | ModelInfo2.Model.PlaneParaInfo.Position3DFited(3,:) = -ModelInfo2.Model.PlaneParaInfo.Position3DFited(3,:); |
---|
| 78 | |
---|
| 79 | % Define possible mathcing point pool |
---|
| 80 | % SurfPoint (surfFeatures that haven't been matches) |
---|
| 81 | % FaceSetPoint (Point that compose the FaceSet in Wrl) |
---|
| 82 | % Specify both Point in image Pixel position and their Index |
---|
| 83 | % (Match index for Surf Features, DepthMap index for FaceSet) |
---|
| 84 | |
---|
| 85 | % initalize FaceSetPoint--------------------- |
---|
| 86 | % ================Min to do: change to FaceSet base not Point base |
---|
| 87 | FaceSetPickedIND1 = setdiff( 1:prod([DepthY1 DepthX1]), ... |
---|
| 88 | find(ModelInfo1.Model.PlaneParaInfo.SupOri == 0)); % used the ful index except the FaceSet Points that will not rendered |
---|
| 89 | % ================================================================ |
---|
| 90 | Ray = ModelInfo1.Model.PlaneParaInfo.Ray; |
---|
| 91 | RR =permute(Ray,[2 3 1]); |
---|
| 92 | temp = RR(:,:,1:2)./repmat(RR(:,:,3),[1 1 2]); |
---|
| 93 | FaceSetPoint1 = permute(temp./repmat(cat(3,Default.a_default,Default.b_default),[Default.VertYNuDepth Default.HoriXNuDepth 1])+... |
---|
| 94 | repmat(cat(3,Default.Ox_default,Default.Oy_default),[Default.VertYNuDepth Default.HoriXNuDepth 1]),[3 1 2]); |
---|
| 95 | FaceSetPoint1(2,:) = 1- FaceSetPoint1(2,:); |
---|
| 96 | FaceSetPoint1 = FaceSetPoint1(:,:).*repmat([Ix1; Iy1],1,length(FaceSetPoint1(:,:))); |
---|
| 97 | %FaceSetPoint1 = FaceSetPoint1(:,FaceSetPickedIND1); |
---|
| 98 | |
---|
| 99 | % ================Min to do: change to FaceSet base not Point base (add July 4th) |
---|
| 100 | FaceSetPickedIND2 = setdiff( 1:prod([DepthY2 DepthX2]), ... |
---|
| 101 | find(ModelInfo2.Model.PlaneParaInfo.SupOri == 0)); % used the ful index except the FaceSet Points that will not rendered |
---|
| 102 | % ================================================================ |
---|
| 103 | Ray = ModelInfo2.Model.PlaneParaInfo.Ray; |
---|
| 104 | RR =permute(Ray,[2 3 1]); |
---|
| 105 | temp = RR(:,:,1:2)./repmat(RR(:,:,3),[1 1 2]); |
---|
| 106 | FaceSetPoint2 = permute(temp./repmat(cat(3,Default.a_default,Default.b_default),[Default.VertYNuDepth Default.HoriXNuDepth 1])+... |
---|
| 107 | repmat(cat(3,Default.Ox_default,Default.Oy_default),[Default.VertYNuDepth Default.HoriXNuDepth 1]),[3 1 2]); |
---|
| 108 | FaceSetPoint2(2,:) = 1- FaceSetPoint2(2,:); |
---|
| 109 | FaceSetPoint2 = FaceSetPoint2(:,:).*repmat([Ix2; Iy2],1,length(FaceSetPoint2(:,:))); |
---|
| 110 | %FaceSetPoint2 = FaceSetPoint2(:,FaceSetPickedIND2); |
---|
| 111 | % ------------------------------------------------ |
---|
| 112 | |
---|
| 113 | % initialize SurfPoint--------------- |
---|
| 114 | if SurfFlag |
---|
| 115 | [SurfPoint1] = readSurf(Img1, defaultPara.Fdir, 'Dense'); % original features |
---|
| 116 | [SurfPoint2] = readSurf(Img2, defaultPara.Fdir, 'Dense'); % original features |
---|
| 117 | |
---|
| 118 | % Modified the ModelInfo to have the structure with the same number of SurfPoint |
---|
| 119 | ModelInfo1_surf = ModelInfo1; |
---|
| 120 | [SurfPoint2DepthMapApproxIND1] = ProjPosi2Mask( [Iy1 Ix1], [DepthY1 DepthX1], SurfPoint1); |
---|
| 121 | x_calib_1 = inv(defaultPara.InrinsicK1)*[ SurfPoint1; ones(1,size(SurfPoint1,2))]; |
---|
| 122 | ray_1 = x_calib_1./repmat(sqrt( sum(x_calib_1.^2,1) ),3,1); |
---|
| 123 | Depth_1 = 1./sum( ModelInfo1.Model.PlaneParaInfo.PlanePara(:,ModelInfo1.Model.PlaneParaInfo.Sup2Para(ModelInfo1.Model.PlaneParaInfo.SupEpand(SurfPoint2DepthMapApproxIND1))).*ray_1, 1); |
---|
| 124 | Position3d_1 = ray_1.*repmat(Depth_1,3,1); |
---|
| 125 | % modified the structure of FitDepth and Position3DFited |
---|
| 126 | ModelInfo1_surf.Model.PlaneParaInfo.FitDepth = Depth_1; |
---|
| 127 | ModelInfo1_surf.Model.PlaneParaInfo.Position3DFited = Position3d_1; |
---|
| 128 | |
---|
| 129 | ModelInfo2_surf = ModelInfo2; |
---|
| 130 | [SurfPoint2DepthMapApproxIND2] = ProjPosi2Mask( [Iy2 Ix2], [DepthY2 DepthX2], SurfPoint2); |
---|
| 131 | x_calib_2 = inv(defaultPara.InrinsicK2)*[SurfPoint2; ones(1,length(SurfPoint2))]; |
---|
| 132 | ray_2 = x_calib_2./repmat(sqrt( sum(x_calib_2.^2,1) ),3,1); |
---|
| 133 | Depth_2 = 1./sum( ModelInfo2.Model.PlaneParaInfo.PlanePara(:,ModelInfo2.Model.PlaneParaInfo.Sup2Para(ModelInfo2.Model.PlaneParaInfo.SupEpand(SurfPoint2DepthMapApproxIND2))).*ray_2, 1); |
---|
| 134 | Position3d_2 = ray_2.*repmat(Depth_2,3,1); |
---|
| 135 | % modified the structure of FitDepth and Position3DFited |
---|
| 136 | ModelInfo2_surf.Model.PlaneParaInfo.FitDepth = Depth_2; |
---|
| 137 | ModelInfo2_surf.Model.PlaneParaInfo.Position3DFited = Position3d_2; |
---|
| 138 | |
---|
| 139 | % get rid of matches SurfFeaturePoints (to aviod conflict in triangulation) |
---|
| 140 | SurfPickedIND1 = setdiff(1:size(SurfPoint1,2), Pair.matches(1,:)); % might use random pick surf Points if there are too many |
---|
| 141 | SurfPickedIND2 = setdiff(1:size(SurfPoint2,2), Pair.matches(2,:)); |
---|
| 142 | %SurfPoint1 = SurfPoint1(:,SurfPickedIND1); |
---|
| 143 | %SurfPoint2 = SurfPoint2(:,SurfPickedIND2); |
---|
| 144 | end |
---|
| 145 | |
---|
| 146 | % Region by picking Img1 |
---|
| 147 | T1_hat = [[0 -Pair.T(3) Pair.T(2)];... |
---|
| 148 | [Pair.T(3) 0 -Pair.T(1)];... |
---|
| 149 | [-Pair.T(2) Pair.T(1) 0]]; |
---|
| 150 | F1 = inv(defaultPara.InrinsicK2)'*T1_hat*Pair.R*inv(defaultPara.InrinsicK1); |
---|
| 151 | if SurfFlag |
---|
| 152 | [POriReprojM1 TargetIND1 OccluedFaceSetIND1 FieldOccluPix1 OcluedDist1 FieldFaceSetIDRemained1] = ... |
---|
| 153 | FindOccluPoint(defaultPara, [Iy2 Ix2], I1, I2, F1, ... |
---|
| 154 | SurfPoint1(:,SurfPickedIND1), FaceSetPoint2(:,FaceSetPickedIND2), SurfPickedIND1, FaceSetPickedIND2, ... |
---|
| 155 | ModelInfo1_surf, ModelInfo2, Pair); |
---|
| 156 | PointPix1 = SurfPoint1(:,TargetIND1); |
---|
| 157 | PointDepth1 = Depth_1(TargetIND1); |
---|
| 158 | % Structure Define: |
---|
| 159 | % POriReprojM1 TargetIND1 OccluedFaceSetIND1 FieldOccluPix1 OcluedDist1 |
---|
| 160 | % -- all of the size that single ray pass through both FaceSet |
---|
| 161 | % OccluedFaceSetIND1 used in DepthMap size |
---|
| 162 | % "Notice" TargetIND1 is im surfFeature size |
---|
| 163 | else |
---|
| 164 | [POriReprojM1 TargetIND1 OccluedFaceSetIND1 FieldOccluPix1 OcluedDist1 FieldFaceSetIDRemained1] = ... |
---|
| 165 | FindOccluPoint(defaultPara, [Iy2 Ix2], I1, I2, F1, ... |
---|
| 166 | FaceSetPoint1(:,FaceSetPickedIND1), FaceSetPoint2(:,FaceSetPickedIND2), FaceSetPickedIND1, FaceSetPickedIND2, ... |
---|
| 167 | ModelInfo1, ModelInfo2, Pair); |
---|
| 168 | PointPix1 = FaceSetPoint1(:,TargetIND1); |
---|
| 169 | PointDepth1 = ModelInfo1.Model.PlaneParaInfo.FitDepth(TargetIND1); |
---|
| 170 | % Structure Define: |
---|
| 171 | % POriReprojM1 TargetIND1 OccluedFaceSetIND1 FieldOccluPix1 OcluedDist1 |
---|
| 172 | % -- all of the size that single ray pass through both FaceSet |
---|
| 173 | % OccluedFaceSetIND1 and TargetIND1 used in DepthMap size |
---|
| 174 | end |
---|
| 175 | save /afs/cs/group/reconstruction3d/scratch/testE/OccluPointFind.mat |
---|
| 176 | % Region by picking Img2 |
---|
| 177 | Pair2.T = -Pair.R'*Pair.T; |
---|
| 178 | Pair2.R = Pair.R'; |
---|
| 179 | Pair2.Xim = Pair.Xim([3 4 1 2],:); |
---|
| 180 | if SurfFlag |
---|
| 181 | [POriReprojM2 TargetIND2 OccluedFaceSetIND2 FieldOccluPix2 OcluedDist2 FieldFaceSetIDRemained2] = ... |
---|
| 182 | FindOccluPoint(defaultPara, [Iy1 Ix1], I2, I1, F1', SurfPoint2(:,SurfPickedIND2), FaceSetPoint1(:,FaceSetPickedIND1), ... |
---|
| 183 | SurfPickedIND2, FaceSetPickedIND1, ModelInfo2_surf, ModelInfo1, Pair2); |
---|
| 184 | PointPix2 = SurfPoint2(:,TargetIND2); |
---|
| 185 | PointDepth2 = Depth_2(TargetIND2); |
---|
| 186 | % Structure Define: |
---|
| 187 | % POriReprojM2 TargetIND2 OccluedFaceSetIND2 FieldOccluPix2 OcluedDist2 |
---|
| 188 | % -- all of the size that single ray pass through both FaceSet |
---|
| 189 | % OccluedFaceSetIND2 used in DepthMap size |
---|
| 190 | % "Notice" TargetIND2 is im surfFeature size |
---|
| 191 | else |
---|
| 192 | [POriReprojM2 TargetIND2 OccluedFaceSetIND2 FieldOccluPix2 OcluedDist2 FieldFaceSetIDRemained2] = ... |
---|
| 193 | FindOccluPoint(defaultPara, [Iy1 Ix1], I2, I1, F1', FaceSetPoint2(:,FaceSetPickedIND2), FaceSetPoint1(:,FaceSetPickedIND1), ... |
---|
| 194 | FaceSetPickedIND2, FaceSetPickedIND1, ModelInfo2, ModelInfo1, Pair2); |
---|
| 195 | PointPix2 = FaceSetPoint2(:,TargetIND2); |
---|
| 196 | PointDepth2 = ModelInfo2.Model.PlaneParaInfo.FitDepth(TargetIND2); |
---|
| 197 | % Structure Define: |
---|
| 198 | %OccluedFaceSetIND2 used in DepthMap size |
---|
| 199 | end |
---|
| 200 | save /afs/cs/group/reconstruction3d/scratch/testE/OccluPointFind.mat |
---|
| 201 | |
---|
| 202 | |
---|
| 203 | % Display |
---|
| 204 | if false %defaultPara.Flag.FindOcclu |
---|
| 205 | MaskPositiveDist1 = OccluDist1 >1; |
---|
| 206 | MaskPositiveDist2 = OccluDist2 >1; |
---|
| 207 | |
---|
| 208 | |
---|
| 209 | figure; |
---|
| 210 | MaxD1 = ones(1, sum(MaskPositiveDist1)); |
---|
| 211 | MinD1 = MaxD1; |
---|
| 212 | MaxD2 = ones(1, sum(MaskPositiveDist2)); |
---|
| 213 | MinD2 = MaxD2; |
---|
| 214 | |
---|
| 215 | dispMatchSearchRegin(I1, I2,[ PointPix1(:,MaskPositiveDist1); ones(1, sum(MaskPositiveDist1))], ... |
---|
| 216 | [ PointPix2(:,MaskPositiveDist2); ones(1, sum(MaskPositiveDist2))], ... |
---|
| 217 | Region1(:,MaskPositiveDist1), Region2(:,MaskPositiveDist2), F1, ... |
---|
| 218 | POriReprojM1(:,MaskPositiveDist1), MaxD1, PoccluM1(:,MaskPositiveDist1), MinD1, ... |
---|
| 219 | POriReprojM2(:,MaskPositiveDist2), MaxD2, PoccluM2(:,MaskPositiveDist2), MinD2, ... |
---|
| 220 | 0, 'Stacking', 'v', 'Interactive', 0); |
---|
| 221 | end |
---|