@inproceedings{SeibtChangLatoschiketal.2024, author = {Seibt, Simon and Chang, Thomas and Latoschik, Marc Erich and Kuth, Bastian and von Rymon Lipinski, Bartosz}, title = {Multidimensional image morphing-fast image-based rendering of open 3D and VR environments}, series = {Virtual Reality \& Intelligent Hardware}, volume = {7}, booktitle = {Virtual Reality \& Intelligent Hardware}, number = {2}, doi = {10.1016/j.vrih.2023.06.007}, pages = {155-172}, year = {2024}, abstract = {In recent years, the demand for interactive photorealistic three-dimensional (3D) environments has increased in various fields, including architecture, engineering, and entertainment. However, achieving a balance between the quality and efficiency of high-performance 3D applications and virtual reality (VR) remains challenging. Methods This study addresses this issue by revisiting and extending view interpolation for image-based rendering (IBR), which enables the exploration of spacious open environments in 3D and VR. Therefore, we introduce multimorphing, a novel rendering method based on the spatial data structure of 2D image patches, called the image graph. Using this approach, novel views can be rendered with up to six degrees of freedom using only a sparse set of views. The rendering process does not require 3D reconstruction of the geometry or per-pixel depth information, and all relevant data for the output are extracted from the local morphing cells of the image graph. The detection of parallax image regions during preprocessing reduces rendering artifacts by extrapolating image patches from adjacent cells in real-time. In addition, a GPU-based solution was presented to resolve exposure inconsistencies within a dataset, enabling seamless transitions of brightness when moving between areas with varying light intensities. Results Experiments on multiple real-world and synthetic scenes demonstrate that the presented method achieves high "VR-compatible" frame rates, even on mid-range and legacy hardware, respectively. While achieving adequate visual quality even for sparse datasets, it outperforms other IBR and current neural rendering approaches. Conclusions Using the correspondence-based decomposition of input images into morphing cells of 2D image patches, multidimensional image morphing provides high-performance novel view generation, supporting open 3D and VR environments. Nevertheless, the handling of morphing artifacts in the parallax image regions remains a topic for future research.}, language = {en} } @inproceedings{ChangSeibtvonRymonLipinski2024, author = {Chang, Thomas and Seibt, Simon and von Rymon Lipinski, Bartosz}, title = {Hierarchical Histogram Threshold Segmentation - Auto-terminating High-detail Oversegmentation}, series = {2024 IEEE/CVF Conference on Computer Vision and Pattern Recognition (CVPR)}, booktitle = {2024 IEEE/CVF Conference on Computer Vision and Pattern Recognition (CVPR)}, doi = {10.1109/CVPR52733.2024.00308}, pages = {3195 -- 3204}, year = {2024}, abstract = {Superpixels play a crucial role in image processing by partitioning an image into clusters of pixels with similar visual attributes. This facilitates subsequent image processing tasks, offering computational advantages over the manipulation of individual pixels. While numerous oversegmentation techniques have emerged in recent years, many rely on predefined initialization and termination criteria. In this paper, a novel top-down superpixel segmentation algorithm called Hierarchical Histogram Threshold Segmentation (HHTS) is introduced. It eliminates the need for initialization and implements auto-termination, outperforming state-of-the-art methods w.r.t. boundary recall. This is achieved by iteratively partitioning individual pixel segments into foreground and background and applying intensity thresholding across multiple color channels. The underlying iterative process constructs a superpixel hierarchy that adapts to local detail distributions until color information exhaustion. Experimental results demonstrate the superiority of the proposed approach in terms of boundary adherence, while maintaining competitive runtime performance on the BSDS500 and NYUV2 datasets. Furthermore, an application of HHTS in refining machine learningbased semantic segmentation masks produced by the Segment Anything Foundation Model (SAM) is presented.}, language = {en} } @misc{SeibtChangvonRymonLipinskietal.2024, author = {Seibt, Simon and Chang, Thomas and von Rymon Lipinski, Bartosz and Latoschik, Marc Erich}, title = {Dense 3D Gaussian Splatting Initialization for Sparse Image Data}, doi = {10.2312/egp.20241038}, pages = {2}, year = {2024}, abstract = {This paper presents advancements in novel-view synthesis with 3D Gaussian Splatting (3DGS) using a dense and accurate SfM point cloud initialization approach. We address the challenge of achieving photorealistic renderings from sparse image data, where basic 3DGS training may result in suboptimal convergence, thus leading to visual artifacts. The proposed method enhances precision and density of initially reconstructed point clouds by refining 3D positions and extrapolating additional points, even for difficult image regions, e.g. with repeating patterns and suboptimal visual coverage. Our contributions focus on improving "Dense Feature Matching for Structure-from-Motion" (DFM4SfM) based on a homographic decomposition of the image space to support 3DGS training: First, a grid-based feature detection method is introduced for DFM4SfM to ensure a welldistributed 3D Gaussian initialization uniformly over all depth planes. Second, the SfM feature matching is complemented by a geometric plausibility check, priming the homography estimation and thereby improving the initial placement of 3D Gaussians. Experimental results on the NeRF-LLFF dataset demonstrate that this approach achieves superior qualitative and quantitative results, even for fewer views, and the potential for a significantly accelerated 3DGS training with faster convergence.}, language = {en} } @inproceedings{vonRymonLipinskiSeibtRothetal.2019, author = {von Rymon Lipinski, Bartosz and Seibt, Simon and Roth, Johannes and Ab{\´e}, Dominik}, title = {Level Graph - Incremental Procedural Generation of Indoor Levels using Minimum Spanning Trees}, doi = {10.1109/CIG.2019.8847956}, pages = {1-7}, year = {2019}, abstract = {Procedural generation of virtual worlds is an important aspect of game development since decades, typically for increasing replayability or for speeding up the level creation process. However, the utilization of this potential has always been a great challenge due to the difficult controllability of the underlying algorithms or limitations to specific level geometries, like 2D regular structures. In this paper, we present a novel approach for semi-automatic generation of a wide variety of 2D/3D corridor and room systems. The underlying processing pipeline is based on a separation between a user-guided generation of a graph-based abstract level structure and a fully-automatic construction of the corresponding geometry using a pre-modeled component library. The core algorithm is built on the computation of an extended minimal spanning tree, which can be controlled by a set of intuitive vertex and edge parameters. First experimental results have shown that our incremental generation pipeline allows the efficient creation of complex indoor levels, minimizing limitations on level and game designers' creativity.}, language = {en} } @inproceedings{SeibtvonRymonLipinskiChangetal.2023, author = {Seibt, Simon and von Rymon Lipinski, Bartosz and Chang, Thomas and Latoschik, Marc Erich}, title = {DFM4SFM - Dense Feature Matching for Structure from Motion}, series = {2023 IEEE International Conference on Image Processing Challenges and Workshops (ICIPCW)}, booktitle = {2023 IEEE International Conference on Image Processing Challenges and Workshops (ICIPCW)}, doi = {10.1109/ICIPC59416.2023.10328368}, pages = {3678 -- 3682}, year = {2023}, abstract = {Structure from motion (SfM) is a fundamental task in computer vision and allows recovering the 3D structure of a stationary scene from an image set. Finding robust and accurate feature matches plays a crucial role in the early stages of SfM. So in this work, we propose a novel method for computing image correspondences based on dense feature matching (DFM) using homographic decomposition: The underlying pipeline provides refinement of existing matches through iterative rematching, detection of occlusions and extrapolation of additional matches in critical image areas between image pairs. Our main contributions are improvements of DFM specifically for SfM, resulting in global refinement and global extrapolation of image correspondences between related views. Furthermore, we propose an iterative version of the Delaunay-triangulation-based outlier detection algorithm for robust processing of repeated image patterns. Through experiments, we demonstrate that the proposed method significantlv improves the reconstruction accuracy.}, language = {en} } @inproceedings{SeibtAroldvonRymonLipinskietal.2024, author = {Seibt, Simon and Arold, Michael and von Rymon Lipinski, Bartosz and Wienkopf, Uwe and Latoschik, Marc Erich}, title = {Parallax-Aware Image Stitching Based on Homographic Decomposition}, publisher = {Springer}, address = {Cham}, isbn = {978-3-031-54605-1}, doi = {10.1007/978-3-031-54605-1_13}, year = {2024}, abstract = {Image stitching plays a crucial role for various computer vision applications, like panoramic photography, video production, medical imaging and satellite imagery. It makes it possible to align two images captured at different views onto a single image with a wider field of view. However, for 3D scenes with high depth complexity and images captured from two different positions, the resulting image pair may exhibit significant parallaxes. Stitching images with multiple or large apparent motion shifts remains a challenging task, and existing methods often fail in such cases. In this paper, a novel image stitching pipeline is introduced, addressing the aforementioned challenge: First, iterative dense feature matching is performed, which results in a multi-homography decomposition. Then, this output is used to compute a per-pixel multidimensional weight map of the estimated homographies for image alignment via weighted warping. Additionally, the homographic image space decomposition is exploited using combinatorial analysis to identify parallaxes, resulting in a parallax-aware overlapping region: Parallax-free overlapping areas only require weighted warping and blending. For parallax areas, these operations are omitted to avoid ghosting artifacts. Instead, histogram- and mask-based color mapping is performed to ensure visual color consistency. The presented experiments demonstrate that the proposed method provides superior results regarding precision and handling of parallaxes.}, language = {en} } @article{LatoschikVONRYMONLIPINSKISeibt2022, author = {Latoschik, Marc Erich and VON RYMON LIPINSKI, BARTOSZ and Seibt, Simon}, title = {Dense Feature Matching Based on Homographic Decomposition}, series = {IEEE Access}, volume = {10}, journal = {IEEE Access}, doi = {10.1109/ACCESS.2022.3152539}, pages = {99129-99149}, year = {2022}, abstract = {Finding robust and accurate feature matches is a fundamental problem in computer vision. However, incorrect correspondences and suboptimal matching accuracies lead to significant challenges for many real-world applications. In conventional feature matching, corresponding features in an image pair are greedily searched using their descriptor distance. The resulting matching set is then typically used as input for geometric model fitting methods to find an appropriate fundamental matrix and filter out incorrect matches. Unfortunately, this basic approach cannot solve all practical problems, such as fundamental matrix degeneration, matching ambiguities caused by repeated patterns and rejection of initially mismatched features without further reconsideration. In this paper we introduce a novel matching pipeline, which addresses all of the aforementioned challenges at once: First, we perform iterative rematching to give mismatched feature points a further chance for being considered in later processing steps. Thereby, we are searching for inliers that exhibit the same homographic transformation per iteration. The resulting homographic decomposition is used for refining matches, occlusion detection (e.g. due to parallaxes) and extrapolation of additional features in critical image areas. Furthermore, Delaunay triangulation of the matching set is utilized to minimize the repeated pattern problem and to implement focused matching. Doing so, enables us to further increase matching quality by concentrating on local image areas, defined by the triangular mesh. We present and discuss experimental results with multiple real-world matching datasets. Our contributions, besides improving matching recall and precision for image processing applications in general, also relate to use cases in image-based computer graphics.}, language = {en} }