@thesis{Oezsahin2020, author = {{\"O}zsahin, Tunahan}, title = {Evaluation of the visualization grammar Vega as a base for visualizing and managing logstash pipelines based on the ELK-Stack}, publisher = {Technische Hochschule Ingolstadt}, address = {Ingolstadt}, url = {http://nbn-resolving.de/urn:nbn:de:bvb:573-8416}, pages = {63, XL}, school = {Technische Hochschule Ingolstadt}, year = {2020}, abstract = {The open-source data processing engine Logstash is a popular tool to build pipelines. It allows building pipelines with real-time processing capabilities for collecting huge volumes and varieties of data from a wide variety of input sources. Logstash can process this data with its built-in plugins and store it in a wide variety of destinations [1]. The pipelines are configured in a declarative grammar and are composed of three main parts, that is, Inputs, Filters, and Outputs. By creating multiple pipelines and defining pipeline to pipeline connections with conditional terms, it is possible to declare complex event processing pipelines. Our goal was to find a graph-based visualization solution to better understand the processing flow and the run time behavior of pipeline to pipeline connections. Currently, there are no official tools or solutions for displaying these connections. At first, we tested the visualization grammar Vega, which is an experimental Kibana built in plugin capable of building visualizations with higher complexity in a declarative language. We searched for a solution, which should be provided by the ELK-Stack, and due to Vega's shortcomings, we evaluated other graph-based visualization frameworks outside of the ELK-Stack as alternatives to find a possible solution for our problem. Based on our findings, we implemented a prototype with the popular JavaScript library React, and a JavaScript framework for visual programming called Rete.js. We implemented a graphical visualization approach that displays the Input, Filter, and Output section of a pipeline as nodes. These nodes are connected with other nodes based on a node to node (edge) connection, which is derived from Logstash pipeline configurations. By creating a visually organized layout of nodes and node to node connections, we made it more convenient to view complex pipeline structures compared to the declarative grammar. In the following, we will outline the actual state of declaring pipelines with Logstash and describe the evaluation of the above-mentioned frameworks, that ultimately led to the implementation of a visualization prototype with React and Rete.js, which gave us important insights as a base for a further implementation such as the development of a Kibana plugin.}, language = {en} }