[37] | 1 | % * This code was used in the following articles:
|
---|
| 2 | % * [1] Learning 3-D Scene Structure from a Single Still Image,
|
---|
| 3 | % * Ashutosh Saxena, Min Sun, Andrew Y. Ng,
|
---|
| 4 | % * In ICCV workshop on 3D Representation for Recognition (3dRR-07), 2007.
|
---|
| 5 | % * (best paper)
|
---|
| 6 | % * [2] 3-D Reconstruction from Sparse Views using Monocular Vision,
|
---|
| 7 | % * Ashutosh Saxena, Min Sun, Andrew Y. Ng,
|
---|
| 8 | % * In ICCV workshop on Virtual Representations and Modeling
|
---|
| 9 | % * of Large-scale environments (VRML), 2007.
|
---|
| 10 | % * [3] 3-D Depth Reconstruction from a Single Still Image,
|
---|
| 11 | % * Ashutosh Saxena, Sung H. Chung, Andrew Y. Ng.
|
---|
| 12 | % * International Journal of Computer Vision (IJCV), Aug 2007.
|
---|
| 13 | % * [6] Learning Depth from Single Monocular Images,
|
---|
| 14 | % * Ashutosh Saxena, Sung H. Chung, Andrew Y. Ng.
|
---|
| 15 | % * In Neural Information Processing Systems (NIPS) 18, 2005.
|
---|
| 16 | % *
|
---|
| 17 | % * These articles are available at:
|
---|
| 18 | % * http://make3d.stanford.edu/publications
|
---|
| 19 | % *
|
---|
| 20 | % * We request that you cite the papers [1], [3] and [6] in any of
|
---|
| 21 | % * your reports that uses this code.
|
---|
| 22 | % * Further, if you use the code in image3dstiching/ (multiple image version),
|
---|
| 23 | % * then please cite [2].
|
---|
| 24 | % *
|
---|
| 25 | % * If you use the code in third_party/, then PLEASE CITE and follow the
|
---|
| 26 | % * LICENSE OF THE CORRESPONDING THIRD PARTY CODE.
|
---|
| 27 | % *
|
---|
| 28 | % * Finally, this code is for non-commercial use only. For further
|
---|
| 29 | % * information and to obtain a copy of the license, see
|
---|
| 30 | % *
|
---|
| 31 | % * http://make3d.stanford.edu/publications/code
|
---|
| 32 | % *
|
---|
| 33 | % * Also, the software distributed under the License is distributed on an
|
---|
| 34 | % * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either
|
---|
| 35 | % * express or implied. See the License for the specific language governing
|
---|
| 36 | % * permissions and limitations under the License.
|
---|
| 37 | % *
|
---|
| 38 | % */
|
---|
| 39 | function [ImgInfo] = ReInference(defaultPara, R, T, Xim, Model, TriDepth, GroundLevel, CoordinateFromRef, FlagRender) |
---|
| 40 | |
---|
| 41 | % This function run the inference in two phase |
---|
| 42 | % Phase 1: Propagate triangulate points info |
---|
| 43 | % Phase 2: Refine the non-triangulated Sup to Vertical and Horizontal setting |
---|
| 44 | % and the multiple image overlap information |
---|
| 45 | |
---|
| 46 | % Input: |
---|
| 47 | % R - rotaion from coordinate ImgInfo(2) to ImgInfo(1) |
---|
| 48 | % T - translation from coordinate ImgInfo(2) to ImgInfo(1) |
---|
| 49 | % Xim - matches in pixel coordinate |
---|
| 50 | % Model - depth previous constrains |
---|
| 51 | % TriDepth - triangulated info |
---|
| 52 | % GroundLevel - ground level info (force ground to have the same level) |
---|
| 53 | % CoordinateFromRef - Rotaion and translation ( [3x3 3x1]) from ImgInfo(1) to the world reference (the first image)a |
---|
| 54 | % FlagRender - rendering the vrml or not |
---|
| 55 | % Return: |
---|
| 56 | % ImgInfo - add constrain; |
---|
| 57 | % |
---|
| 58 | |
---|
| 59 | % initialize parameters ----------------------------------------------------- |
---|
| 60 | scale = 1; |
---|
| 61 | StickHori = 5; %0.1; % sticking power in horizontal direction |
---|
| 62 | StickVert = 5; % sticking power in vertical direction |
---|
| 63 | Center = 10; % Co-Planar weight at the Center of each superpixel |
---|
| 64 | TriangulatedWeight = 20; |
---|
| 65 | GroundLevelWright = 500; |
---|
| 66 | GroundWeight = 20; |
---|
| 67 | VertWeight = 10 |
---|
| 68 | HoriConf = 1; % set the confidant of the learned depth at the middle in Horizontal direction of the image |
---|
| 69 | VertConf = 0.01; % set the confidant of the learned depth at the top of the image |
---|
| 70 | [ VertYNuDepth HoriXNuDepth] = size(Model(1).Depth.FitDepth); |
---|
| 71 | mapVert = linspace(VertConf,1,VertYNuDepth); % modeling the gravity prior |
---|
| 72 | mapHori = [linspace(HoriConf,1,round(HoriXNuDepth/2)) fliplr(linspace(HoriConf,1,HoriXNuDepth-round(HoriXNuDepth/2)))]; |
---|
| 73 | % ========set the range of depth that our model in |
---|
| 74 | ClosestDist = defaultPara.Closestdist; |
---|
| 75 | FarestDist = defaultPara.FarestDist; % //Min: do not need it since depths are all been processed |
---|
| 76 | % ================================================ |
---|
| 77 | ceiling = 0*VertYNuDepth; % set the position of the ceiling, related to No plane coming back constrain % changed for newchurch |
---|
| 78 | if ~isempty(Model(1).MultiScaleSupTable) |
---|
| 79 | MultiScaleFlag = true; |
---|
| 80 | WeiV = 2*ones(1,size(Model(1).MultiScaleSupTable,2)-1); |
---|
| 81 | else |
---|
| 82 | MultiScaleFlag = false; |
---|
| 83 | WeiV = 1; |
---|
| 84 | end |
---|
| 85 | WeiV(1,1:2:end) = 6; % emphasize the middle scale three times smaller than large scale |
---|
| 86 | WeiV = WeiV./sum(WeiV);% normalize if pair of superpixels have same index in all the scale, their weight will be 10 |
---|
| 87 | ShiftStick = -.1; % between -1 and 0, more means more smoothing. |
---|
| 88 | ShiftCoP = -.5; % between -1 and 0, more means more smoothing. |
---|
| 89 | gravity =true; % if true, apply the HoriConf and VertConf linear scale weight |
---|
| 90 | % ======================================= |
---|
| 91 | groundThreshold = cos([ zeros(1, VertYNuDepth - ceil(VertYNuDepth/2)+10) ... |
---|
| 92 | linspace(0,15,ceil(VertYNuDepth/2)-10)]*pi/180); |
---|
| 93 | % v1 15 v2 20 too big v3 20 to ensure non misclassified as ground. |
---|
| 94 | % verticalThreshold = cos(linspace(5,55,Default.VertYNuDepth)*pi/180); % give a vector of size 55 in top to down : |
---|
| 95 | verticalThreshold = cos([ 5*ones(1,VertYNuDepth - ceil(VertYNuDepth/2)) ... |
---|
| 96 | linspace(5,55,ceil(VertYNuDepth/2))]*pi/180); |
---|
| 97 | % give a vector of size 55 in top to down : |
---|
| 98 | % 50 means suface norm away from y axis more than 50 degree |
---|
| 99 | % =========================================================================================================================================== |
---|
| 100 | if strcmp(defaultPara.InitialDepth,'FitDepth') |
---|
| 101 | CleanedDepthMap = Model(1).Depth.FitDepth; |
---|
| 102 | else |
---|
| 103 | CleanedDepthMap = Model(1).Depth.RawDepth; |
---|
| 104 | end |
---|
| 105 | |
---|
| 106 | % ----------------------------------------------------------------------------------- |
---|
| 107 | |
---|
| 108 | % Pre-Processing -------------------------------------------------------------------- |
---|
| 109 | % Clean the Sup near sky |
---|
| 110 | maskSky = Model(1).Sup == 0; |
---|
| 111 | maskSkyEroded = imerode(maskSky, strel('disk', 4) ); |
---|
| 112 | SupEpand = ExpandSup2Sky(Sup,maskSkyEroded); |
---|
| 113 | NuPatch = HoriXNuDepth*VertYNuDepth-sum(maskSky(:)); |
---|
| 114 | NuSup = setdiff(unique(Sup)',0); |
---|
| 115 | NuSup = sort(NuSup); |
---|
| 116 | NuSupSize = size(NuSup,2); |
---|
| 117 | % Sup index and planeParameter index inverse map |
---|
| 118 | Sup2Para = sparse(1,max(Sup(:))); |
---|
| 119 | Sup2Para(NuSup) = 1:NuSupSize; |
---|
| 120 | |
---|
| 121 | Posi3D = im_cr2w_cr(Model(1).Depth.FitDepth, permute(Model(1).Ray,[2 3 1])); % (3 by VertNuDepth HoriNuDepth) |
---|
| 122 | % ----------------------------------------------------------------------------------- |
---|
| 123 | |
---|
| 124 | % Generate the Matrix for MRF ------------------------------------------------------- |
---|
| 125 | CleanedDepthMap = Model(1).Depth.FitDepth; |
---|
| 126 | PosiM = sparse(0,0); % Position matrix: first self term objective |
---|
| 127 | RayAllM = sparse(0,0); % all the Ray for regular grid to set depth constrain |
---|
| 128 | ScalingTerm = sparse( 0, 1); |
---|
| 129 | YPointer = []; |
---|
| 130 | YPosition = []; |
---|
| 131 | beta = []; |
---|
| 132 | for i = NuSup |
---|
| 133 | mask = SupEpand ==i; % include the Ray that will be use to expand the NonSky |
---|
| 134 | RayAllM = blkdiag( RayAllM, Ray(:,mask)'); |
---|
| 135 | mask = Sup ==i; % Not include the Ray that will be use to expand the NonSky |
---|
| 136 | [yt x] = find(mask); |
---|
| 137 | CenterX = round(median(x)); |
---|
| 138 | CenterY = round(median(yt)); |
---|
| 139 | YPointer = [YPointer; CenterY >= ceiling]; % Y is zero in the top ceiling default to be 0 as the top row in the image |
---|
| 140 | YPosition = [YPosition; CenterY]; |
---|
| 141 | mask(isnan(CleanedDepthMap)) = false; |
---|
| 142 | SupNuPatch(i) = sum(mask(:)); |
---|
| 143 | % find center point |
---|
| 144 | [yt x] = find(mask); |
---|
| 145 | CenterX = round(median(x)); |
---|
| 146 | CenterY = round(median(yt)); |
---|
| 147 | |
---|
| 148 | if ~all(mask(:)==0) |
---|
| 149 | if gravity |
---|
| 150 | if any(CleanedDepthMap(mask) <=0) |
---|
| 151 | CleanedDepthMap(mask) |
---|
| 152 | end |
---|
| 153 | PosiM = blkdiag(PosiM,Posi3D(:,mask)');%.*repmat( mapVert(yt)',[1 3]).*repmat( mapHori(x)',[1 3])); |
---|
| 154 | if SupMatched == i |
---|
| 155 | ScalingTerm = [ScalingTerm; ones( sum(mask(:)), 1)]; |
---|
| 156 | else |
---|
| 157 | ScalingTerm = [ScalingTerm; zeros( sum(mask(:)), 1)]; |
---|
| 158 | end |
---|
| 159 | else |
---|
| 160 | PosiM = blkdiag(PosiM,Posi3D(:,mask)'); |
---|
| 161 | if SupMatched == i |
---|
| 162 | ScalingTerm = [ScalingTerm; ones( sum(mask(:)), 1)]; |
---|
| 163 | else |
---|
| 164 | ScalingTerm = [ScalingTerm; zeros( sum(mask(:)), 1)]; |
---|
| 165 | end |
---|
| 166 | end |
---|
| 167 | else |
---|
| 168 | PosiM = blkdiag(PosiM, Posi3D(:,mask)'); |
---|
| 169 | end |
---|
| 170 | end |
---|
| 171 | YPointer(YPointer==0) = -1; |
---|
| 172 | |
---|
| 173 | % =================Building up the triangulated and sampled ground constrain ========================== |
---|
| 174 | PosiTriangulatedM = sparse( 0, 3*NuSupSize); |
---|
| 175 | count = 1; |
---|
| 176 | for i = SupMatched' |
---|
| 177 | temp = sparse(1, 3*NuSupSize); |
---|
| 178 | if Sup2Para(i)*3>3*NuSupSize || Sup2Para(i)*3-2<0 |
---|
| 179 | Sup2Para(i) |
---|
| 180 | NuSupSize |
---|
| 181 | count = count + 1; |
---|
| 182 | continue; |
---|
| 183 | end |
---|
| 184 | % Sup2Para(i) |
---|
| 185 | % i |
---|
| 186 | temp( (Sup2Para(i)*3-2):(Sup2Para(i)*3) ) = RayMatched(count,:)*ClosestDepth(count); |
---|
| 187 | PosiTriangulatedM = [PosiTriangulatedM; temp]; |
---|
| 188 | count = count + 1; |
---|
| 189 | end |
---|
| 190 | PosiSampledGroundM = sparse( 0, 3*NuSupSize); |
---|
| 191 | count = 1; |
---|
| 192 | for i = SampledGroundSupMatched' |
---|
| 193 | temp = sparse(1, 3*NuSupSize); |
---|
| 194 | if (Sup2Para(i)*3)>(3*NuSupSize) || (Sup2Para(i)*3-2)<0 |
---|
| 195 | Sup2Para(i) |
---|
| 196 | NuSupSize |
---|
| 197 | end |
---|
| 198 | temp( (Sup2Para(i)*3-2):(Sup2Para(i)*3) ) = SampledGroundRayMatched(count,:)*SampledGroundClosestDepth(count); |
---|
| 199 | PosiSampledGroundM = [PosiSampledGroundM; temp]; |
---|
| 200 | count = count + 1; |
---|
| 201 | end |
---|
| 202 | |
---|
| 203 | % ================================================================================================ |
---|
| 204 | |
---|
| 205 | |
---|