1 | % * This code was used in the following articles:
|
---|
2 | % * [1] Learning 3-D Scene Structure from a Single Still Image,
|
---|
3 | % * Ashutosh Saxena, Min Sun, Andrew Y. Ng,
|
---|
4 | % * In ICCV workshop on 3D Representation for Recognition (3dRR-07), 2007.
|
---|
5 | % * (best paper)
|
---|
6 | % * [2] 3-D Reconstruction from Sparse Views using Monocular Vision,
|
---|
7 | % * Ashutosh Saxena, Min Sun, Andrew Y. Ng,
|
---|
8 | % * In ICCV workshop on Virtual Representations and Modeling
|
---|
9 | % * of Large-scale environments (VRML), 2007.
|
---|
10 | % * [3] 3-D Depth Reconstruction from a Single Still Image,
|
---|
11 | % * Ashutosh Saxena, Sung H. Chung, Andrew Y. Ng.
|
---|
12 | % * International Journal of Computer Vision (IJCV), Aug 2007.
|
---|
13 | % * [6] Learning Depth from Single Monocular Images,
|
---|
14 | % * Ashutosh Saxena, Sung H. Chung, Andrew Y. Ng.
|
---|
15 | % * In Neural Information Processing Systems (NIPS) 18, 2005.
|
---|
16 | % *
|
---|
17 | % * These articles are available at:
|
---|
18 | % * http://make3d.stanford.edu/publications
|
---|
19 | % *
|
---|
20 | % * We request that you cite the papers [1], [3] and [6] in any of
|
---|
21 | % * your reports that uses this code.
|
---|
22 | % * Further, if you use the code in image3dstiching/ (multiple image version),
|
---|
23 | % * then please cite [2].
|
---|
24 | % *
|
---|
25 | % * If you use the code in third_party/, then PLEASE CITE and follow the
|
---|
26 | % * LICENSE OF THE CORRESPONDING THIRD PARTY CODE.
|
---|
27 | % *
|
---|
28 | % * Finally, this code is for non-commercial use only. For further
|
---|
29 | % * information and to obtain a copy of the license, see
|
---|
30 | % *
|
---|
31 | % * http://make3d.stanford.edu/publications/code
|
---|
32 | % *
|
---|
33 | % * Also, the software distributed under the License is distributed on an
|
---|
34 | % * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either
|
---|
35 | % * express or implied. See the License for the specific language governing
|
---|
36 | % * permissions and limitations under the License.
|
---|
37 | % *
|
---|
38 | % */
|
---|
39 | function [ImgInfo] = ReInference(defaultPara, R, T, Xim, Model, TriDepth, GroundLevel, CoordinateFromRef, FlagRender) |
---|
40 | |
---|
41 | % This function run the inference in two phase |
---|
42 | % Phase 1: Propagate triangulate points info |
---|
43 | % Phase 2: Refine the non-triangulated Sup to Vertical and Horizontal setting |
---|
44 | % and the multiple image overlap information |
---|
45 | |
---|
46 | % Input: |
---|
47 | % R - rotaion from coordinate ImgInfo(2) to ImgInfo(1) |
---|
48 | % T - translation from coordinate ImgInfo(2) to ImgInfo(1) |
---|
49 | % Xim - matches in pixel coordinate |
---|
50 | % Model - depth previous constrains |
---|
51 | % TriDepth - triangulated info |
---|
52 | % GroundLevel - ground level info (force ground to have the same level) |
---|
53 | % CoordinateFromRef - Rotaion and translation ( [3x3 3x1]) from ImgInfo(1) to the world reference (the first image)a |
---|
54 | % FlagRender - rendering the vrml or not |
---|
55 | % Return: |
---|
56 | % ImgInfo - add constrain; |
---|
57 | % |
---|
58 | |
---|
59 | % initialize parameters ----------------------------------------------------- |
---|
60 | scale = 1; |
---|
61 | StickHori = 5; %0.1; % sticking power in horizontal direction |
---|
62 | StickVert = 5; % sticking power in vertical direction |
---|
63 | Center = 10; % Co-Planar weight at the Center of each superpixel |
---|
64 | TriangulatedWeight = 20; |
---|
65 | GroundLevelWright = 500; |
---|
66 | GroundWeight = 20; |
---|
67 | VertWeight = 10 |
---|
68 | HoriConf = 1; % set the confidant of the learned depth at the middle in Horizontal direction of the image |
---|
69 | VertConf = 0.01; % set the confidant of the learned depth at the top of the image |
---|
70 | [ VertYNuDepth HoriXNuDepth] = size(Model(1).Depth.FitDepth); |
---|
71 | mapVert = linspace(VertConf,1,VertYNuDepth); % modeling the gravity prior |
---|
72 | mapHori = [linspace(HoriConf,1,round(HoriXNuDepth/2)) fliplr(linspace(HoriConf,1,HoriXNuDepth-round(HoriXNuDepth/2)))]; |
---|
73 | % ========set the range of depth that our model in |
---|
74 | ClosestDist = defaultPara.Closestdist; |
---|
75 | FarestDist = defaultPara.FarestDist; % //Min: do not need it since depths are all been processed |
---|
76 | % ================================================ |
---|
77 | ceiling = 0*VertYNuDepth; % set the position of the ceiling, related to No plane coming back constrain % changed for newchurch |
---|
78 | if ~isempty(Model(1).MultiScaleSupTable) |
---|
79 | MultiScaleFlag = true; |
---|
80 | WeiV = 2*ones(1,size(Model(1).MultiScaleSupTable,2)-1); |
---|
81 | else |
---|
82 | MultiScaleFlag = false; |
---|
83 | WeiV = 1; |
---|
84 | end |
---|
85 | WeiV(1,1:2:end) = 6; % emphasize the middle scale three times smaller than large scale |
---|
86 | WeiV = WeiV./sum(WeiV);% normalize if pair of superpixels have same index in all the scale, their weight will be 10 |
---|
87 | ShiftStick = -.1; % between -1 and 0, more means more smoothing. |
---|
88 | ShiftCoP = -.5; % between -1 and 0, more means more smoothing. |
---|
89 | gravity =true; % if true, apply the HoriConf and VertConf linear scale weight |
---|
90 | % ======================================= |
---|
91 | groundThreshold = cos([ zeros(1, VertYNuDepth - ceil(VertYNuDepth/2)+10) ... |
---|
92 | linspace(0,15,ceil(VertYNuDepth/2)-10)]*pi/180); |
---|
93 | % v1 15 v2 20 too big v3 20 to ensure non misclassified as ground. |
---|
94 | % verticalThreshold = cos(linspace(5,55,Default.VertYNuDepth)*pi/180); % give a vector of size 55 in top to down : |
---|
95 | verticalThreshold = cos([ 5*ones(1,VertYNuDepth - ceil(VertYNuDepth/2)) ... |
---|
96 | linspace(5,55,ceil(VertYNuDepth/2))]*pi/180); |
---|
97 | % give a vector of size 55 in top to down : |
---|
98 | % 50 means suface norm away from y axis more than 50 degree |
---|
99 | % =========================================================================================================================================== |
---|
100 | if strcmp(defaultPara.InitialDepth,'FitDepth') |
---|
101 | CleanedDepthMap = Model(1).Depth.FitDepth; |
---|
102 | else |
---|
103 | CleanedDepthMap = Model(1).Depth.RawDepth; |
---|
104 | end |
---|
105 | |
---|
106 | % ----------------------------------------------------------------------------------- |
---|
107 | |
---|
108 | % Pre-Processing -------------------------------------------------------------------- |
---|
109 | % Clean the Sup near sky |
---|
110 | maskSky = Model(1).Sup == 0; |
---|
111 | maskSkyEroded = imerode(maskSky, strel('disk', 4) ); |
---|
112 | SupEpand = ExpandSup2Sky(Sup,maskSkyEroded); |
---|
113 | NuPatch = HoriXNuDepth*VertYNuDepth-sum(maskSky(:)); |
---|
114 | NuSup = setdiff(unique(Sup)',0); |
---|
115 | NuSup = sort(NuSup); |
---|
116 | NuSupSize = size(NuSup,2); |
---|
117 | % Sup index and planeParameter index inverse map |
---|
118 | Sup2Para = sparse(1,max(Sup(:))); |
---|
119 | Sup2Para(NuSup) = 1:NuSupSize; |
---|
120 | |
---|
121 | Posi3D = im_cr2w_cr(Model(1).Depth.FitDepth, permute(Model(1).Ray,[2 3 1])); % (3 by VertNuDepth HoriNuDepth) |
---|
122 | % ----------------------------------------------------------------------------------- |
---|
123 | |
---|
124 | % Generate the Matrix for MRF ------------------------------------------------------- |
---|
125 | CleanedDepthMap = Model(1).Depth.FitDepth; |
---|
126 | PosiM = sparse(0,0); % Position matrix: first self term objective |
---|
127 | RayAllM = sparse(0,0); % all the Ray for regular grid to set depth constrain |
---|
128 | ScalingTerm = sparse( 0, 1); |
---|
129 | YPointer = []; |
---|
130 | YPosition = []; |
---|
131 | beta = []; |
---|
132 | for i = NuSup |
---|
133 | mask = SupEpand ==i; % include the Ray that will be use to expand the NonSky |
---|
134 | RayAllM = blkdiag( RayAllM, Ray(:,mask)'); |
---|
135 | mask = Sup ==i; % Not include the Ray that will be use to expand the NonSky |
---|
136 | [yt x] = find(mask); |
---|
137 | CenterX = round(median(x)); |
---|
138 | CenterY = round(median(yt)); |
---|
139 | YPointer = [YPointer; CenterY >= ceiling]; % Y is zero in the top ceiling default to be 0 as the top row in the image |
---|
140 | YPosition = [YPosition; CenterY]; |
---|
141 | mask(isnan(CleanedDepthMap)) = false; |
---|
142 | SupNuPatch(i) = sum(mask(:)); |
---|
143 | % find center point |
---|
144 | [yt x] = find(mask); |
---|
145 | CenterX = round(median(x)); |
---|
146 | CenterY = round(median(yt)); |
---|
147 | |
---|
148 | if ~all(mask(:)==0) |
---|
149 | if gravity |
---|
150 | if any(CleanedDepthMap(mask) <=0) |
---|
151 | CleanedDepthMap(mask) |
---|
152 | end |
---|
153 | PosiM = blkdiag(PosiM,Posi3D(:,mask)');%.*repmat( mapVert(yt)',[1 3]).*repmat( mapHori(x)',[1 3])); |
---|
154 | if SupMatched == i |
---|
155 | ScalingTerm = [ScalingTerm; ones( sum(mask(:)), 1)]; |
---|
156 | else |
---|
157 | ScalingTerm = [ScalingTerm; zeros( sum(mask(:)), 1)]; |
---|
158 | end |
---|
159 | else |
---|
160 | PosiM = blkdiag(PosiM,Posi3D(:,mask)'); |
---|
161 | if SupMatched == i |
---|
162 | ScalingTerm = [ScalingTerm; ones( sum(mask(:)), 1)]; |
---|
163 | else |
---|
164 | ScalingTerm = [ScalingTerm; zeros( sum(mask(:)), 1)]; |
---|
165 | end |
---|
166 | end |
---|
167 | else |
---|
168 | PosiM = blkdiag(PosiM, Posi3D(:,mask)'); |
---|
169 | end |
---|
170 | end |
---|
171 | YPointer(YPointer==0) = -1; |
---|
172 | |
---|
173 | % =================Building up the triangulated and sampled ground constrain ========================== |
---|
174 | PosiTriangulatedM = sparse( 0, 3*NuSupSize); |
---|
175 | count = 1; |
---|
176 | for i = SupMatched' |
---|
177 | temp = sparse(1, 3*NuSupSize); |
---|
178 | if Sup2Para(i)*3>3*NuSupSize || Sup2Para(i)*3-2<0 |
---|
179 | Sup2Para(i) |
---|
180 | NuSupSize |
---|
181 | count = count + 1; |
---|
182 | continue; |
---|
183 | end |
---|
184 | % Sup2Para(i) |
---|
185 | % i |
---|
186 | temp( (Sup2Para(i)*3-2):(Sup2Para(i)*3) ) = RayMatched(count,:)*ClosestDepth(count); |
---|
187 | PosiTriangulatedM = [PosiTriangulatedM; temp]; |
---|
188 | count = count + 1; |
---|
189 | end |
---|
190 | PosiSampledGroundM = sparse( 0, 3*NuSupSize); |
---|
191 | count = 1; |
---|
192 | for i = SampledGroundSupMatched' |
---|
193 | temp = sparse(1, 3*NuSupSize); |
---|
194 | if (Sup2Para(i)*3)>(3*NuSupSize) || (Sup2Para(i)*3-2)<0 |
---|
195 | Sup2Para(i) |
---|
196 | NuSupSize |
---|
197 | end |
---|
198 | temp( (Sup2Para(i)*3-2):(Sup2Para(i)*3) ) = SampledGroundRayMatched(count,:)*SampledGroundClosestDepth(count); |
---|
199 | PosiSampledGroundM = [PosiSampledGroundM; temp]; |
---|
200 | count = count + 1; |
---|
201 | end |
---|
202 | |
---|
203 | % ================================================================================================ |
---|
204 | |
---|
205 | |
---|