@misc{HerglotzOchMeyeretal., author = {Herglotz, Christian and Och, Hannah and Meyer, Anna and Ramasubbu, Geetha and Eicherm{\"u}ller, Lena and Kr{\"a}nzler, Matthias and Brand, Fabian and Fischer, Kristian and Nguyen, Dat Thanh and Regensky, Andy and Kaup, Andr{\´e}}, title = {The Bj{\o}ntegaard Bible Why Your Way of Comparing Video Codecs May Be Wrong}, series = {IEEE Transactions on Image Processing}, journal = {IEEE Transactions on Image Processing}, number = {Volume 33}, issn = {1057-7149}, doi = {10.1109/TIP.2023.3346695}, pages = {987 -- 1001}, abstract = {In this paper, we provide an in-depth assessment on the Bj{\o}ntegaard Delta. We construct a large data set of video compression performance comparisons using a diverse set of metrics including PSNR, VMAF, bitrate, and processing energies. These metrics are evaluated for visual data types such as classic perspective video, 360° video, point clouds, and screen content. As compression technology, we consider multiple hybrid video codecs as well as state-of-the-art neural network based compression methods. Using additional supporting points in-between standard points defined by parameters such as the quantization parameter, we assess the interpolation error of the Bj{\o}ntegaard-Delta (BD) calculus and its impact on the final BD value. From the analysis, we find that the BD calculus is most accurate in the standard application of rate-distortion comparisons with mean errors below 0.5 percentage points. For other applications and special cases, e.g., VMAF quality, energy considerations, or inter-codec comparisons, the errors are higher (up to 5 percentage points), but can be halved by using a higher number of supporting points. We finally come up with recommendations on how to use the BD calculus such that the validity of the resulting BD-values is maximized. Main recommendations are as follows: First, relative curve differences should be plotted and analyzed. Second, the logarithmic domain should be used for saturating metrics such as SSIM and VMAF. Third, BD values below a certain threshold indicated by the subset error should not be used to draw recommendations. Fourth, using two supporting points is sufficient to obtain rough performance estimates.}, language = {en} } @misc{HerglotzKraenzlerChuetal., author = {Herglotz, Christian and Kr{\"a}nzler, Matthias and Chu, Xixue and Fran{\c{c}}ois, Edouard and He, Yong and Kaup, Andr{\´e}}, title = {Extended Signaling Methods for Reduced Video Decoder Power Consumption Using Green Metadata}, series = {IEEE Transactions on Circuits and Systems II: Express Briefs}, volume = {Volume 71}, journal = {IEEE Transactions on Circuits and Systems II: Express Briefs}, number = {Issue 3}, issn = {1549-7747}, doi = {10.1109/TCSII.2023.3328405}, pages = {1141 -- 1145}, abstract = {In this brief, we discuss one aspect of the latest MPEG standard edition on energy-efficient media consumption, also known as Green Metadata (ISO/IEC 232001-11), which is the interactive signaling for remote decoder-power reduction for peer-to-peer video conferencing. In this scenario, the receiver of a video, e.g., a battery-driven portable device, can send a dedicated request to the sender which asks for a video bitstream representation that is less complex to decode and process. Consequently, the receiver saves energy and extends operating times. We provide an overview on latest studies from the literature dealing with energy-saving aspects, which motivate the extension of the legacy Green Metadata standard. Furthermore, we explain the newly introduced syntax elements and verify their effectiveness by performing dedicated experiments. We show that the integration of these syntax elements can lead to dynamic energy savings of up to 90\% for software video decoding and 80\% for hardware video decoding, respectively.}, language = {en} } @misc{RamasubbuKaupHerglotz, author = {Ramasubbu, Geetha and Kaup, Andr{\´e} and Herglotz, Christian}, title = {Modeling the Energy Consumption of the HEVC Software Encoding Process using Processor events}, series = {IEEE 26th International Workshop on Multimedia Signal Processing (MMSP)}, journal = {IEEE 26th International Workshop on Multimedia Signal Processing (MMSP)}, publisher = {IEEE}, isbn = {979-8-3503-8725-4}, issn = {2473-3628}, doi = {10.1109/MMSP61759.2024.10743858}, abstract = {Developing energy-efficient video encoding algorithms is highly important due to the high processing complexities and, consequently, the high energy demand of the encoding process. To accomplish this, the energy consumption of the video encoders must be studied, which is only possible with a complex and dedicated energy measurement setup. This emphasizes the need for simple energy estimation models, which estimate the energy required for the encoding. Our paper investigates the possibility of estimating the energy demand of a HEVC software CPU-encoding process using processor events. First, we perform energy measurements and obtain processor events using dedicated profiling software. Then, by using the measured energy demand of the encoding process and profiling data, we build an encoding energy estimation model that uses the processor events of the ultrafast encoding preset to obtain the energy estimate for complex encoding presets with a mean absolute percentage error of 5.36\% when averaged over all the presets. Additionally, we present an energy model that offers the possibility of obtaining energy distribution among various encoding sub-processes. energy models from literature. By using a unified evaluation framework we show how accurately the required decoding energy for different decoding systems can be approximated. We give thorough explanations on the model parameters and explain how the model variables are derived. To show the modeling capabilities in general, we test the estimation performance for different decoding software and hardware solutions, where we find that the proposed model outperforms the models from literature by reaching frame-wise mean estimation errors of less than 7\% for software and less than 15\% for hardware based systems.}, language = {en} } @misc{HerglotzKraenzlerXuetal., author = {Herglotz, Christian and Kr{\"a}nzler, Matthias and Xu, Bide and Kaup, Andr{\´e}}, title = {Decoding Energy Optimization for Video Coding Using Model-Driven Gradient Descent}, series = {IEEE 26th International Workshop on Multimedia Signal Processing (MMSP)}, journal = {IEEE 26th International Workshop on Multimedia Signal Processing (MMSP)}, isbn = {979-8-3503-8725-4}, issn = {2473-3628}, doi = {10.1109/MMSP61759.2024.10743503}, abstract = {Nowadays, a large part of the global energy consumption caused by video communications can be attributed to end-user devices such as smartphones, tablet PCs, and TV sets. In this paper, we present a method to increase the performance of an existing algorithm dedicated to reduce the end-user side energy consumption during video streaming. The algorithm, which is called decoding-energy-rate-distortion optimization (DERDO), exploits a decoding energy model during encoding and chooses coding modes in such a way that the software decoding energy is minimized. In this paper, we develop a dedicated gradient descent approach for DERDO that refines specific energy coefficients used for decoding energy modeling. We find that this approach boosts the performance of DERDO by increasing the energy savings by at least 5\% with respect to standard DERDO. As a consequence, we observe decoding energy savings of more than 40\% and more than 7\% for practical encoder and decoder implementations of HEVC and H.264/AVC, respectively, when compared to standard encoding using classic rate-distortion optimization.}, language = {en} } @misc{ZargariaslHerglotz, author = {Zargariasl, Hamid and Herglotz, Christian}, title = {Impact of Topology Manipulation on Digital Thread Functionality: A Case Study on Aerospace Engineering}, series = {9th International Conference on Smart and Sustainable Technologies (SpliTech)}, journal = {9th International Conference on Smart and Sustainable Technologies (SpliTech)}, publisher = {IEEE}, isbn = {978-953-290-135-1}, doi = {10.23919/SpliTech61897.2024.10612419}, abstract = {The emergence of Industry 4.0 has necessitated the integration of technologies like Internet of Things (IoT) concepts into the production, operation, and processes of industries and institutions. Industry with the growing amount of data generated in facilities, digital models such as digital twin and the digital thread representing the communication between the devices can be leveraged to efficiently execute and monitor the processes. Linking different contributing assets to their counterpart digital twins is one of the main roles of the digital thread. Creating a digital thread for any product or the set of operations to manage a procedure during the life cycle of a product has been a major focus in the last few years. It eliminates a lot of difficulties (such as errors and delays) and optimizes the operations in every stage of designing products, services, and so on. Considering the numerous advantages of implementing the digital thread, there is still space to further improve the technology. Digital thread, as an infrastructure of data flow, can be optimized by mitigating problems such as latency in the communication between the interconnected nodes. This thread of communication between nodes can be shown as a mathematical graph made of vertices and edges so that the set of edges realizes the digital thread concept. Viewing a digital thread as a graph emerges the idea of exploiting well-known graph theory knowledge to figure out the problems of communication. In this paper, a graph representation of the digital thread is analyzed and one candidate scenario for reducing the network distance, which is the average path connecting two network nodes, will be studied as a use case. This approach intends to optimize the performance with latency as the key performance metric, by manipulating network distance as an independent variable.}, language = {en} } @misc{RueckertUllmannHerglotzetal., author = {R{\"u}ckert, Rainer and Ullmann, Ingrid and Herglotz, Christian and Kaup, Andr{\´e} and Vossiek, Martin}, title = {Data Compression for Close-Range Radar Imaging}, series = {IEEE Transactions on Radar Systems}, journal = {IEEE Transactions on Radar Systems}, issn = {2832-7357}, doi = {10.1109/TRS.2024.3387288}, pages = {421 -- 433}, abstract = {The resolution of radar images is constantly increasing. As a result, radar images require more storage space, which is associated with increased costs. Therefore, it is advantageous to minimize the data size. In this paper, we present various compression methods for reducing the data size of radar images. Compression and decompression are performed in two scenarios. In the first scenario, the raw data are compressed and decompressed before the image is reconstructed. In the second scenario, the reconstructed image itself is compressed and decompressed. In both scenarios, the reconstructed radar image is compared with the original image. Due to its widespread use, High-Efficiency Video Coding (HEVC) is used as a state-of-the-art benchmark for both scenarios and compared with proprietary algorithms that combine lossy and lossless compression. A discrete Fourier transform-based compression algorithm from the automotive sector is used as another state-of-the-art benchmark. This is applied against our novel approaches, which are based on the discrete cosine transform, use of direct thresholding in the spatial domain, or are applied to the maximum intensity projection. With the exception of HEVC, all algorithms presented have in common that they perform lossy data processing in the first step and then use the Lempel-Ziv-Markov algorithm as a lossless compression step. To compare the compression ratios, we use various image- and video-specific metrics, such as the peak signal-to-noise ratio (PSNR), the similarity of speeded-up robust features, and the structural similarity index measure (SSIM). For a simple classification, we use Otsu's method to examine the effects of compression on the images. The radar images are categorized into transparent and nontransparent based on the measurement objects. Depending on the application and the desired resolution, our approaches can achieve storage savings of up to 99.93 \% compared to the uncompressed data with PSNR and SSIM values of 38.8 dB and 0.916, respectively.}, language = {en} } @misc{EichermuellerChaudhariKatsavounidisetal., author = {Eicherm{\"u}ller, Lena and Chaudhari, Gaurang and Katsavounidis, Ioannis and Lei, Zhijun and Tmar, Hassene and Herglotz, Christian and Kaup, Andr{\´e}}, title = {Encoding Time and Energy Model for SVT-AV1 Based on Video Complexity}, series = {ICASSP 2024 - 2024 IEEE International Conference on Acoustics, Speech and Signal Processing (ICASSP)}, journal = {ICASSP 2024 - 2024 IEEE International Conference on Acoustics, Speech and Signal Processing (ICASSP)}, publisher = {IEEE}, isbn = {979-8-3503-4485-1}, issn = {2379-190X}, doi = {10.1109/ICASSP48485.2024.10446602}, pages = {3370 -- 3374}, abstract = {The share of online video traffic in global carbon dioxide emissions is growing steadily. To comply with the demand for video media, dedicated compression techniques are continuously optimized, but at the expense of increasingly higher computational demands and thus rising energy consumption at the video encoder side. In order to find the best trade-off between compression and energy consumption, modeling encoding energy for a wide range of encoding parameters is crucial. We propose an encoding time and energy model for SVT-AV1 based on empirical relations between the encoding time and video parameters as well as encoder configurations. Furthermore, we model the influence of video content by established content descriptors such as spatial and temporal information. We then use the predicted encoding time to estimate the required energy demand and achieve a prediction error of 19.6\% for encoding time and 20.9\% for encoding energy.}, language = {en} } @misc{EichermuellerChaudhariKatsavounidisetal., author = {Eicherm{\"u}ller, Lena and Chaudhari, Gaurang and Katsavounidis, Ioannis and Lei, Zhijun and Tmar, Hassene and Herglotz, Christian and Kaup, Andr{\´e}}, title = {SVT-AV1 Encoding Bitrate Estimation Using Motion Search Information}, series = {32nd European Signal Processing Conference (EUSIPCO)}, journal = {32nd European Signal Processing Conference (EUSIPCO)}, publisher = {IEEE}, isbn = {978-9-4645-9361-7}, issn = {2076-1465}, pages = {937 -- 941}, abstract = {Enabling high compression efficiency while keeping encoding energy consumption at a low level, requires prioritization of which videos need more sophisticated encoding techniques. However, the effects vary highly based on the content, and information on how good a video can be compressed is required. This can be measured by estimating the encoded bitstream size prior to encoding. We identified the errors between estimated motion vectors from Motion Search, an algorithm that predicts temporal changes in videos, correlates well to the encoded bitstream size. Combining Motion Search with Random Forests, the encoding bitrate can be estimated with a Pearson correlation of above 0.96.}, language = {en} } @misc{StuerzenhofaeckerKraenzlerHerglotzetal., author = {St{\"u}rzenhof{\"a}cker, Teresa and Kr{\"a}nzler, Matthias and Herglotz, Christian and Kaup, Andr{\´e}}, title = {Design Space Exploration at Frame-Level for Joint Decoding Energy and Quality Optimization in VVC}, series = {2024 32nd European Signal Processing Conference (EUSIPCO)}, journal = {2024 32nd European Signal Processing Conference (EUSIPCO)}, publisher = {IEEE}, isbn = {978-9-4645-9361-7}, issn = {2076-1465}, pages = {932 -- 936}, abstract = {In the pursuit of a reduced energy demand of VVC decoders, it was found that the coding tool configuration has a substantial influence on the bit rate efficiency and the decoding energy demand. The Advanced Design Space Exploration algorithm as proposed in the literature, can derive coding tool configurations that provide optimal trade-offs between rate and energy efficiency. Yet, some trade-off points in the design space cannot be reached with the state-of-the-art methodology, which defines coding tools for an entire bitstream. This work proposes a novel, granular adjustment of the coding tool usage in VVC. Consequently, the optimization algorithm is adjusted to explore coding tool configurations that operate on frame-level. Moreover, new optimization criteria are introduced to focus the search on specific bit rates. As a result, coding tool configurations are obtained which yield so far inaccessible trade-offs between bit rate efficiency and decoding energy demand for VVC-coded sequences. The proposed methodology extends the design space and enhances the continuity of the Pareto front.}, language = {en} } @misc{EichermuellerChaudhariKatsavounidisetal., author = {Eicherm{\"u}ller, Lena and Chaudhari, Gaurang and Katsavounidis, Ioannis and Lei, Zhijun and Tmar, Hassene and Herglotz, Christian and Kaup, Andre}, title = {Encoding Time and Energy Model for SVT-AV1 based on Video Complexity}, series = {arXiv - accepted for IEEE International Conference on Acoustics, Speech, and Signal Processing (ICASSP)}, journal = {arXiv - accepted for IEEE International Conference on Acoustics, Speech, and Signal Processing (ICASSP)}, doi = {10.48550/arXiv.2401.16067}, pages = {5}, abstract = {The share of online video traffic in global carbon dioxide emissions is growing steadily. To comply with the demand for video media, dedicated compression techniques are continuously optimized, but at the expense of increasingly higher computational demands and thus rising energy consumption at the video encoder side. In order to find the best trade-off between compression and energy consumption, modeling encoding energy for a wide range of encoding parameters is crucial. We propose an encoding time and energy model for SVT-AV1 based on empirical relations between the encoding time and video parameters as well as encoder configurations. Furthermore, we model the influence of video content by established content descriptors such as spatial and temporal information. We then use the predicted encoding time to estimate the required energy demand and achieve a prediction error of 19.6 \% for encoding time and 20.9 \% for encoding energy.}, language = {en} } @misc{RamasubbuKaupHerglotz, author = {Ramasubbu, Geetha and Kaup, Andr{\´e} and Herglotz, Christian}, title = {Towards Video Codec Performance Evaluation: A Rate-Energy-Distortion Perspective}, series = {16th International Conference on Quality of Multimedia Experience (QoMEX)}, journal = {16th International Conference on Quality of Multimedia Experience (QoMEX)}, issn = {2472-7814}, doi = {10.1109/QoMEX61742.2024.10598269}, pages = {96 -- 99}, abstract = {The Bj{\o}ntegaard Delta rate (BD-rate) objectively assesses the coding efficiency of video codecs using the rate-distortion (R-D) performance but overlooks encoding energy, which is crucial in practical applications, especially for those on handheld devices. Although R-D analysis can be extended to incorporate encoding energy as energy-distortion (E-D), it fails to integrate all three parameters seamlessly. This work proposes a novel approach to address this limitation by introducing a 3D representation of rate, encoding energy, and distortion through surface fitting. In addition, we evaluate various surface fitting techniques based on their accuracy and investigate the proposed 3D representation and its projections. The overlapping areas in projections help in encoder selection and recommend avoiding the slow presets of the older encoders (x264, x265), as the recent encoders (x265, VVenC) offer higher quality for the same bitrate-energy performance and provide a lower rate for the same energy-distortion performance.}, language = {en} } @misc{AmiriLeMoanHerglotz, author = {Amiri, Mitra and Le Moan, Steven and Herglotz, Christian}, title = {Exploiting Change Blindness for Video Coding: Perspectives from a Less Promising User Study}, series = {16th International Conference on Quality of Multimedia Experience (QoMEX)}, journal = {16th International Conference on Quality of Multimedia Experience (QoMEX)}, publisher = {IEEE}, issn = {2472-7814}, doi = {10.1109/QoMEX61742.2024.10598281}, abstract = {What the human visual system can perceive is strongly limited by the capacity of our working memory and attention. Such limitations result in the human observer's inability to perceive large-scale changes in a stimulus, a phenomenon known as change blindness. In this paper, we started with the premise that this phenomenon can be exploited in video coding, especially HDR-video compression where the bitrate is high. We designed an HDR-video encoding approach that relies on spatially and temporally varying quantization parameters within the framework of HEVC video encoding. In the absence of a reliable change blindness prediction model, to extract compression candidate regions (CCR) we used an existing saliency prediction algorithm. We explored different configurations and carried out a subjective study to test our hypothesis. While our methodology did not lead to significantly superior performance in terms of the ratio between perceived quality and bitrate, we were able to determine potential flaws in our methodology, such as the employed saliency model for CCR prediction (chosen for computational efficiency, but eventually not sufficiently accurate), as well as a very strong subjective bias due to observers priming themselves early on in the experiment about the type of artifacts they should look for, thus creating a scenario with little ecological validity.}, language = {en} } @misc{KraenzlerHerglotzKaup, author = {Kr{\"a}nzler, Matthias and Herglotz, Christian and Kaup, Andr{\´e}}, title = {A Comprehensive Review of Software and Hardware Energy Efficiency of Video Decoders}, series = {Picture Coding Symposium (PCS) 2024}, journal = {Picture Coding Symposium (PCS) 2024}, publisher = {IEEE}, issn = {2472-7822}, doi = {10.1109/PCS60826.2024.10566363}, abstract = {Energy and compression efficiency are two essential parts of modern video decoder implementations that have to be considered. This work comprehensively studies the following six video coding formats regarding compression and decoding energy efficiency: AVC, VP9, HEVC, AV1, VVC, and AVM. We first evaluate the energy demand of reference and optimized software decoder implementations. Furthermore, we consider the influence of the usage of SIMD instructions on those decoder implementations. We find that AV1 is a sweet spot for optimized software decoder implementations with an additional energy demand of 16.55\% and bitrate savings of -43.95\% compared to VP9. We furthermore evaluate the hardware decoding energy demand of four video coding formats. Thereby, we show that AV1 has energy demand increases by 117.50\% compared to VP9. For HEVC, we found a sweet spot in terms of energy demand with an increase of 6.06\% with respect to VP9. Relative to their optimized software counterparts, hardware video decoders reduce the energy consumption to less than 9\% compared to software decoders}, language = {en} } @misc{HerglotzKraenzlerDaietal., author = {Herglotz, Christian and Kr{\"a}nzler, Matthias and Dai, Rui and Kaup, Andr{\´e}}, title = {Complexity Metrics for VVC Decoder Power Reduction in Green Metadata}, series = {Picture Coding Symposium (PCS) 2024}, journal = {Picture Coding Symposium (PCS) 2024}, publisher = {IEEE}, isbn = {979-8-3503-5848-3}, issn = {2472-7822}, doi = {10.1109/PCS60826.2024.10566419}, pages = {1 -- 5}, abstract = {This paper discusses the new complexity metrics for VVC decoder power reduction introduced in the 3 rd edition of the Green Metadata standard. The standard defines dedicated syntax elements that represent the expected software decoding complexity with a high accuracy. Using a simple complexity model, which can be trained for any VVC decoder software implementation, the receiver of the video can estimate the processing complexity to decode the subsequent video segment. Afterwards, it can adjust the clock frequency of the processor to keep the real-time playback constraint. By reducing the frequency of the processor, a significant amount of energy is saved. In this paper, we present the syntax elements, their meaning, and show that they can accurately estimate the processing complexity of various software decoder implementations with errors below 11\%. Furthermore, we present a processor-frequency-control algorithm and apply it to a development board performing VVC video decoding. Measurements reveal that the complexity metric signaling can lead to up to 30\% of energy savings.}, language = {en} } @misc{LeMoanAmiriHerglotz, author = {Le Moan, Steven and Amiri, Mitra and Herglotz, Christian}, title = {Exploiting Change Blindness to Reduce Bitrate and Display Luminance in Video Streaming}, series = {IEEE International Conference on Image Processing (ICIP)}, journal = {IEEE International Conference on Image Processing (ICIP)}, publisher = {IEEE}, isbn = {979-8-3503-4939-9}, issn = {2381-8549}, doi = {10.1109/ICIP51287.2024.10648096}, abstract = {This paper investigates the potential of exploiting change blindness (CB), a perceptual phenomenon where changes in a visual stimulus are not noticed by the observer, to enhance rendering efficiency and achieve higher compression gains in video encoding. We explore various distortion techniques, including foveation, cropping, and dimming, to introduce changes that can be noticed but typically are not, due to the limited bandwidth of our perceptual experience. Through a user study involving HEVC-encoded videos and a task designed to direct gaze, we assess the impact of these distortions on perceived video quality, bitrate reduction, and display luminance. Our findings suggest that significant bitrate and luminance reductions can be achieved without adversely affecting perceived quality, highlighting CB's potential for reducing energy consumption and strain on streaming infrastructure. Despite observer variability and the ephemeral nature of CB, our results demonstrate that conscious attention plays a crucial role in the perception of video quality and that exploiting CB can lead to substantial efficiency gains in video coding.}, language = {en} } @misc{HerglotzLeMoanMercat, author = {Herglotz, Christian and Le Moan, Steven and Mercat, Alexandre}, title = {Energy Reduction Opportunities in HDR Video Encoding}, series = {IEEE International Conference on Image Processing (ICIP)}, journal = {IEEE International Conference on Image Processing (ICIP)}, isbn = {979-8-3503-4939-9}, issn = {2381-8549}, doi = {10.1109/ICIP51287.2024.10647527}, abstract = {This paper investigates the energy consumption of video encoding for high dynamic range videos. Specifically, we compare the energy consumption of the compression process using 10 -bit input sequences, a tone-mapped 8 -bit input sequence at 10 -bit internal bit depth, and encoding an 8 -bit input sequence using an encoder with an internal bit depth of 8 bit. We find that linear scaling of the luminance and chrominance values leads to degradations of the visual quality, but that significant encoding complexity and thus encoding energy can be saved. An important reason for this is the availability of vector instructions, which are not available for the 10-bit encoder. Furthermore, we find that at sufficiently low target bitrates, the compression efficiency at an internal bit depth of 8 bit exceeds the compression efficiency of regular 10-bit encoding.}, language = {en} }