\documentclass[12pt, a4paper, oneside, titlepage, listof=totoc, headinclude, open=right, glossaries=totoc, captions=tableheading, final, BCOR=10mm, parskip=never]{paper} \usepackage[english]{babel} \usepackage[utf8]{inputenc} \usepackage[T1]{fontenc} \usepackage{textcomp} \usepackage[usenames,dvipsnames,rgb,svgnames,table]{xcolor} \definecolor{osc-out}{RGB}{150,0,255} \definecolor{osc-in}{RGB}{0,0,255} \definecolor{audio-in}{RGB}{255,0,0} \definecolor{audio-out}{RGB}{0,206,0} \definecolor{pubsub-in}{RGB}{128,0,0} \definecolor{controller-in}{RGB}{0,128,0} \definecolor{table-background-one}{RGB}{194,195,194} \definecolor{table-background-two}{RGB}{210,213,210} \usepackage[cache=false,newfloat=true]{minted} \usepackage{mdframed} \usepackage{fancyhdr} %\usepackage{ulem} %\usepackage{lmodern} \usepackage{url} \usepackage{graphicx} \usepackage{pdfpages} \usepackage[author={David Runge}]{pdfcomment} \usepackage{mathtools} \usepackage{float} \floatstyle{boxed} \restylefloat{figure} \usepackage{tocloft} \setcounter{secnumdepth}{4} \usepackage{hyperref} \hypersetup{hidelinks, colorlinks = false} \usepackage[font=scriptsize]{caption} \usepackage[authoryear]{natbib} % glossary \usepackage[acronym,nonumberlist,toc,xindy]{glossaries} \newacronym{aap}{AAP}{Ambisonics Amplitude Panning} \newacronym{adat}{ADAT}{Alesis Digital Audio Tape} \newacronym{alsa}{ALSA}{Advanced Linux Sound Architecture} \newacronym{apf}{APF}{Audio Processing Framework} \newacronym{api}{API}{Application programming interface} \newacronym{asdf}{ASDF}{Audio Scene Description Format} \newacronym{bs}{BS}{Binaural Synthesis} \newacronym{brs}{BRS}{Binaural Room Synthesis} \newacronym{cc}{CC}{Creative Commons} \newacronym{cicm}{CICM}{Centre de recherche Informatique et Création Musicale} \newacronym{cnmat}{CNMAT}{Center for New Music and Audio Technologies} \newacronym{cpu}{CPU}{Central Processing Unit} \newacronym{fdl}{FDL}{GNU Free Documentation License} \newacronym{gpl}{GPL}{GNU General Public License} \newacronym{gui}{GUI}{Graphical user interface} \newacronym{lgpl}{LGPL}{GNU Lesser General Public License} \newacronym{lts}{LTS}{Long Term Support} \newacronym{hoa}{HOA}{Higher Order Ambisonics} \newacronym{ip}{IP}{Internet Protocol} \newacronym{jack}{JACK}{JACK Audio Connection Kit} \newacronym{madi}{MADI}{Multichannel Audio Digital Interface} \newacronym{nfc-hoa}{NFC-HOA}{Near-field-corrected Higher Order Ambisonics} \newacronym{oop}{OOP}{Object-oriented Programming} \newacronym{os}{OS}{Operating System} \newacronym{osc}{OSC}{Open Sound Control} \newacronym{posix}{POSIX}{Portable Operating System Interface} \newacronym{pubsub}{PubSub}{Publish-subscribe message pattern} \newacronym{pd}{Pd}{PureData} \newacronym{raii}{RAII}{Ressource acquisition is initialization} \newacronym{ssr}{SSR}{SoundScape Renderer} \newacronym{tcp}{TCP}{Transmission Control Protocol} \newacronym{udp}{UDP}{User Datagram Protocol} \newacronym{vbap}{VBAP}{Vector Based Amplitude Panning} \newacronym{wfs}{WFS}{Wave Field Synthesis} \newacronym{xml}{XML}{Extensible Markup Language} \newglossaryentry{sclang}{ name={sclang}, description={Name of the SuperCollider programming language and the interpreter executable of the SuperCollider programming language} } \newglossaryentry{id}{ name={ID}, description={A name or number, that identifies an object}, plural=IDs } \newglossaryentry{stdout}{ name={stdout}, description={The standard output is a stream where a program writes its output data to. This can be a log file or a terminal}, } \makeglossaries \graphicspath{{../images//}} \begin{document} \pagenumbering{Roman} \pagestyle{empty} \begin{titlepage} \centering \includegraphics[width=0.3\textwidth]{tu-berlin-logo.pdf}\par\vspace{1cm} {\scshape\LARGE Technische Universität Berlin\par} \vspace{1cm} {\scshape\Large Master Thesis\par} \vspace{1.5cm} {\huge\bfseries A Networking Extension for the SoundScape Renderer\par} \vspace{2cm} {\Large\itshape David Runge\par} \vspace{1.5cm} {\large (340592)}\\ \href{dave@sleepmap.de}{dave@sleepmap.de}\\ \vfill supervised by\par Henrik von Coler and Stefan Weinzierl \vfill {\large \today\par} \end{titlepage} \section*{Eidesstattliche Erklärung} \vspace{1cm} Hiermit erkläre ich, dass ich die vorliegende Arbeit selbstständig und eigenhändig sowie ohne unerlaubte fremde Hilfe und ausschließlich unter Verwendung der aufgeführten Quellen und Hilfsmittel angefertigt habe.\\ Berlin, den \today\par\\ \vspace{2cm} \noindent\ldots\ldots\ldots\ldots\ldots\ldots\ldots\ldots\ldots\ldots\ldots\\ David Runge \newpage Thanks \newpage \begin{abstract} \gls{wfs} as a technological concept has been around for many years now and all over the world several institutions run small and some even large scale setups ranging from single speaker lines to those facilitating a couple of hundred loudspeakers respectively.\\ The still evolving implementations are driven by several rendering engines, of which two free and open-source ones, namely sWONDER and SoundScape Renderer, have (partially) been developed at TU Berlin.\\ The latter due to its current design is not yet able to render for large scale setups, ie.\ those using several computers to render audio on a loudspeaker setup, due to the high amount of channels.\\ Its solid codebase however, which additionally offers a framework for many more renderering types, and the ongoing development, deems further work on this application a good future investment.\\ This work is about the extension of the SoundScape Renderer's functionality to turn it into a networking application for large scale \gls{wfs} setups. \end{abstract} \newpage \renewcommand{\abstractname}{Zusammenfassung} \begin{abstract} \gls{wfs} as a technological concept has been around for many years now and all over the world several institutions run small and some even large scale setups ranging from single speaker lines to those facilitating a couple of hundred loudspeakers respectively.\\ The still evolving implementations are driven by several rendering engines, of which two free and open-source ones, namely sWONDER and SoundScape Renderer, have (partially) been developed at TU Berlin.\\ The latter due to its current design is not yet able to render for large scale setups, ie.\ those using several computers to render audio on a loudspeaker setup, due to the high amount of channels.\\ Its solid codebase however, which additionally offers a framework for many more renderering types, and the ongoing development, deems further work on this application a good future investment.\\ This work is about the extension of the SoundScape Renderer's functionality to turn it into a networking application for large scale \gls{wfs} setups. \end{abstract} \newpage \setcounter{page}{1} \pagestyle{plain} \printglossaries \newpage \phantomsection \addcontentsline{toc}{section}{\listtablename} \listoftables \newpage \phantomsection \addcontentsline{toc}{section}{\listfigurename} \listoffigures \newpage \phantomsection \addcontentsline{toc}{section}{\listlistingname} \listoflistings \newpage \pagestyle{empty} { \setcounter{tocdepth}{4} \renewcommand{\thispagestyle}[1]{} \tableofcontents \newpage } \pagestyle{headings} \setcounter{page}{1} \pagenumbering{arabic} \renewcommand{\theFancyVerbLine}{\textcolor{gray}{\scriptsize\arabic{FancyVerbLine}}} \RecustomVerbatimEnvironment{Verbatim}{Verbatim}{xleftmargin=5mm} \section{Introduction} \label{sec:introduction} \cleardoublepage \section{Free and open-source spatial audio renderers} \label{sec:freespatialaudiorenderers} To date there exist five (known of) free and open-source spatial audio renderers, which are all \gls{jack} \citep{website:jackaudio2016} clients: \begin{itemize} \item sWONDER \citep{website:swonder2016}, developed by Technische Universität Berlin, Germany \item WFSCollider \citep{website:wfscollider2016}, developed by \href{http://gameoflife.nl/en}{Game Of Life Foundation} \citep{website:gameoflife2016}, The Hague, Netherlands \item HoaLibrary for \gls{pd} \citep{website:hoalibraryforpd} developed at \gls{cicm}, Paris, France \item 3Dj for SuperCollider \citep{thesis:perezlopez3dj2014}, developed at Universitat Pompeu Fabra, Barcelona \item \gls{ssr} \citep{website:ssr2016}, developed by Quality \& Usability Lab, Telekom Innovation Laboratories, TU Berlin and Institut für Nachrichtentechnik, Universität Rostock and Division of Applied Acoustics, Chalmers University of Technology \end{itemize} Different concepts and contexts apply to all of the renderers, which are about to be explained briefly in the following sections, prefixed by a section about spatial audio renderer appliances and followed by one about free software and its pitfalls. \subsection{Spatial audio renderers and their appliance} \label{subsec:spatialaudiorenderersandtheirappliance} \subsubsection{Wave Field Synthesis} \label{subsubsec:wavefieldsynthesis} \gls{wfs} describes a spatial technique for rendering audio. As such it aims at synthesizing a sound field of desired acoustic preference in a given listening area, assuming a planar reproduction to be most suitable for most applications.\\ \gls{wfs} is typically implemented using a curved or linear loudspeaker array surrounding the listening area.\\ Several free and open-source renderer applications exist for \gls{wfs} environments, with varying stages of feature richness.\\ The proposed work will focus on one of them and its extension towards \gls{wfs} on large scale systems. \subsubsection{Higher order ambisonics and vector based amplitude panning} \label{subsubsec:hoaandvbap} \subsubsection{Binaural (Room) Synthesis} \label{subsubsec:binaural} \subsection{sWONDER} \label{subsec:swonder} sWONDER \citep{baalman2007} consists of a set of C++ applications that provide \gls{bs} and \gls{wfs} rendering. In 2007 it was specifically redesigned \citep{baalmanetal2007} to cope with large scale \gls{wfs} setups in which several (computer) nodes, providing several speakers each, drive a system together.\\ In these setups each node receives all available audio streams (which represent one virtual audio source respectively) redundantly and a master application signals which node is responsible for rendering what source on which speaker.\\ It uses \gls{osc} for messaging between its parts and for setting its controls. Apart from that, it can be controlled through a \gls{gui}, that was specifically designed for it. Unfortunately sWONDER has not been actively maintained for several years, has a complex setup chain and many bugs, that are not likely to get fixed any time soon. \subsection{HoaLibrary (PureData extension)} \label{subsec:hoalibrary_puredata_extension} \subsection{3Dj (SuperCollider Quark)} \label{subsec:3dj_supercollider_quark} \subsection{WFSCollider} \label{subsec:wfscollider} WFSCollider was built on top of \href{https://supercollider.github.io}{SuperCollider} 3.5 \citep{website:supercollider} and is also capable of driving large scale systems. It uses a different approach in doing so, though: Whereas with sWONDER all audio streams are distributed to each node, WFSCollider usually uses the audio files to be played on all machines simultaneously and synchronizes between them.\\ It has a feature-rich \gls{gui} in the \textit{many window} style, making available time lines and movement of sources through facilitating what the \gls{sclang} has to offer.\\ As WFSCollider basically is SuperCollider plus extra features, it is also an \gls{osc} enabled application and can thus also be used for mere multi-channel playback of audio.\\ Although it has many useful features, it requires MacOSX (Linux version still untested) to run, is built upon a quite old version of \href{https://supercollider.github.io}{SuperCollider} and is likely never to be merged into it, due to many core changes to it. \subsection{SoundScape Renderer} \label{subsec:soundscaperenderer} \gls{ssr}, also a C++ application, running on Linux and MacOSX, is a multi-purpose spatial audio renderer, as it is not only capable of \gls{bs} and \gls{wfs}, but also \gls{hoa} and \gls{vbap}.\\ It can be used with a \gls{gui} or headless (without one), depicting the virtual sources, their volumes and positions, alongside which speakers are currently used for rendering a selected source. \gls{ssr} uses TCP/IP sockets for communication and is therefore not directly \gls{osc} enabled. This functionality can be achieved using the capapilities of other applications such as \gls{pd} \citep{website:puredata2016} in combination with it though.\\ Unlike the two renderers above, the \gls{ssr} is not able to run large-scale \gls{wfs} setups, as it lacks the features to communicate between instances of itself on several computers, while these instances serve a subset of the available loudspeakers. \subsection{Why free software matters and what its pitfalls are} \label{subsec:whyfreesoftwarematters} Free software is the terminology for software published under a free license. Licenses, such as the \gls{gpl} are considered free, because they allow for anyone to copy, modify and redistribute the source code (under the same license).\\ Research is a field of work, in which reproducability is very important, as findings need to be independently verifiable. Scientific work is published and shared (sometimes also under free licenses, such as \gls{cc}) amongst research groups of institutions all around the world. In an ideal world, all scientific research would be published under a free documentation license, such as the \gls{fdl}, allowing access to anyone.\\ The software used in scientific institutions is unfortunately rarely free (e.g. Microsoft Exchange, Microsoft Office, Adobe Photoshop in general and Steinberg's Cubase, Cycling '74's Max/MSP, Avid's ProTools or Ableton Live in the audio production field) and additionally mostly bound to proprietary \glspl{os}, such as Microsoft Windows or Apple's MacOS, preventing interoperability, development and an open society.\\ Free software however enables students and researchers to learn from the source code directly (if they can and want to), to modify (and improve) it and to share their findings. More than with proprietary software, it is possible to have a community develop around it, that takes care of the project for a long time.\\ Free software nonetheless can not be considered a silver bullet. It is after all only a way of developing software and not a way to grade its efficiency or code quality. Additionally, especially in a scientific context it can happen, that software is conceived by an institution, put to use, but afterwards not developed any further, as the programmers go their way (e.g. \nameref{subsec:swonder}). This way a high responsibility weighs on these institutions, as they need to ensure further development on systems, not easily accessible to the public, or not feasible for home use (e.g. \gls{wfs}). This situation however also holds a great opportunity for coorporation. \cleardoublepage \section{Implementation} \label{sec:implementation} This section covers the implementation of a networking interface for the \gls{ssr} and the considerations leading to it. The application was chosen for extension, as it offers a wide set of rendering algorithms (in various stages of completion) by using the \gls{apf} \citep{MatthiasGeierTorbenHohn1890} and because it was still actively maintained by its creators at the time of writing.\\ \subsection{Outline} \label{subsec:outline} Initially it was aimed at extending the \gls{ssr}'s features in the scope of creating a replacement for the aging sWONDER software, enabling it to run networked instances to drive a medium or large scale \gls{wfs} setup. The approach appeared too narrow however, as the application offers many different rendering algorithms. A networking extension therefore would have to be available to all of them with an equal feature set. Additionally, extending a rendering framework by a networking feature, with the help of only one of its engines proved to be linked to a massive, but avoidable overhead (see~\ref{subsubsec:preliminaries}).\\ The \gls{ssr}, being a multi-purpose spatial audio renderer, can be used in diverse setup scenarios (see~\ref{subsubsec:setups}), which is why not only classic server-client relationships (see~\ref{subsubsec:remote_controlling_a_server}), but also client-only and local (see~\ref{subsubsec:remote_controlling_a_client}) setups had to be taken account of. In addition the case of medium and large scale loudspeaker based rendering setups and their specifics had to be considered (see~\ref{subsubsec:rendering_on_dedicated_speakers}). \subsubsection{Prelimenaries} \label{subsubsec:preliminaries} In preparation to this work, an implemention of a side-by-side installation to the \gls{os} currently driving the \gls{wfs} system setup of the Electronic Studio at TU Berlin \citep{website:tu-electronic_studio} was attempted for testing purposes.\\ Arch Linux \citep{website:archlinux} was installed and configured to run the medium scale setup. Unfortunately the proprietary Dante \citep{website:audinate} driver for Linux, offered by Four Audio \citep{website:fouraudio}, creates non-trivial circumstances for using it on an up-to-date Linux kernel, due to \gls{alsa} \gls{api} changes not accounted for.\\ While the current \gls{os} - an Ubuntu \citep{website:ubuntu} Studio 2012 \gls{lts} - still runs well in its own parameters, its support has run out and it is therefore becoming harder, if not impossible, to build newer software on it, using newer versions of free software compilers.\\ For research purposes however, it is desirable to be able to try new kernel and software features, finding the most stable and secure setup possible involving realtime enabled kernels and building new versions of (spatialisation) software on a regular basis.\\ The hardware of the large scale setup at TU Berlin in room H0104 was being updated and unusable at the time of writing. In the future it will however become a valuable candidate for testing of the sought after \gls{ssr} features, as its setup involves no Dante network, but is instead run by several rendering computers connected to \gls{madi} and \gls{adat} lightpipe enabled speaker systems.\\ Although a \gls{wfs} setup for testing purposes was eligible, it generally is not needed for implementing the features described in the following sections and subsections, as they can be tested using two machines running Linux, \gls{jack} and a development version of the \gls{ssr}.\\ \subsubsection{Remote controlling a server} \label{subsubsec:remote_controlling_a_server} A \gls{ssr} server instance in the notion of a medium to large scale reproduction setup is supposed to have a set of \textit{n} (pre-)assigned clients. Generally, controlling it should be possible through either \gls{udp} or \gls{tcp}. Every message sent to it, should be distributed to all of its clients (if applicable), preferably using the same protocol used to communicate with the server. The messaging system should be flexible and scriptable.\\ All audio inputs available to the server should be available to its clients as well. A server instance should be able to render audio just as a client would. It should be able to receive messages from its clients and act upon them (e.g.\ updating \gls{gui} elements). \subsubsection{Remote controlling clients} \label{subsubsec:remote_controlling_a_client} A \gls{ssr} client can either be local (on the same machine) or somewhere on the same network, as the server or application controlling it. It should not make a difference, if the client instance is controlled by a server instance or any other application, implementing the messaging system it uses. A client should send an update to its server or the application controlling it, upon receiving a valid message. \subsubsection{Rendering on dedicated speakers} \label{subsubsec:rendering_on_dedicated_speakers} In a medium or large scale setup, \textit{n} clients collectively render audio on \textit{l} loudspeakers, while all should be using the same \textit{i} inputs and each have \textit{c} hardware outputs. \textit{l} is preferably a multiple of \textit{c}, but definitely larger than \textit{c}.\\ As the described setups usually have too many loudspeakers for only one machine (i.e.\ a client), a system has to be conceived, by which each client will only render on its specifically assigned subset of size \textit{c} of the \textit{n} loudspeakers. \subsection{Publisher/Subscriber interface} \label{subsec:publisher_subscriber_interface} The \gls{ssr} internally uses a \gls{pubsub}, which is a design pattern, to implement control through and over several parts of its components.\\ In \gls{oop} \gls{pubsub} - also called observer, listener messaging - is usually comprised of a publisher class, handling the messages, without explicitely implementing how they will be used and a subscriber class, that allows for its implementations to subscribe to the messages provided. Filtering takes place to enable subscribers to only receive a certain subset of the messages.\\ The \gls{ssr} implements a content-based filtering system, in which each subscriber evaluates the messages received and acts depending on its own constraints to implement further actions upon it.\\ \begin{figure}[!htb] \centering \includegraphics[scale=0.6, trim = 6mm 130mm 12mm 4mm, clip] {ssr-publisher-with-legacy-subscribers.pdf} \caption{A diagram depicting a simplified version of the \gls{pubsub} used within the \gls{ssr}.\\ {\color{pubsub-in}\textbf{--}} Calls from Publisher to Subscriber {\color{controller-in}\textbf{--}} Calls from Subscribers to Controller (Publisher) } \label{fig:ssr-publisher-with-legacy-subscribers} \end{figure} The abstract class Publisher defines the messages possible to send and provides means to subscribe to them. The global Controller class is its only implementation within the \gls{ssr}.\\ The abstract class Subscriber in turn defines the messages understood, while its implementations in RenderSubscriber, Scene, OscSender and NetworkSubscriber take care of how they are used.\\ This system enables a versatile messaging layout, in which components can call the publisher functionality in Controller, which in turn will send out messages to all of its subscribers.\\ Depending on the design of an application, \gls{pubsub} is not necesarily a one-way-road. As showin in Figure~\ref{fig:ssr-publisher-with-legacy-subscribers}, subscribers can also be able to call functions of the Publisher, if the implementation permits it.\\ In the \gls{ssr} this is possible, because each Subscriber holds a reference to the Controller instance and is therefore able to call its public functions.\\ According to the principle of encapsulation in \gls{oop}, this type of functionality is handled by a separate class. In Figure~\ref{fig:ssr-publisher-with-legacy-subscribers}, the OscHandler and Server instances delegate calls to Controller functionality to their utilities OscReceiver and CommandParser (respectively). \subsection{IP interface} \label{subsec:ip-interface} The \gls{ssr} from early on incorporated a network interface, that accepts specially terminated \gls{xml}-formatted strings over a \gls{tcp} port, called “\gls{ip} interface”. This has the benefit of reusing the same \gls{xml} parser code in use for scene and reproduction description.\\ A downside is however, that - from the perspective of other software - it is complicated to use, as a conversion to \gls{xml} has to be attempted before sending a message to the \gls{ssr}. Additionally the message has to be linted (error checked) before sending and again parsed, after receiving an answer from the application.\\ The \gls{ip} interface achieves to offer more or less direct access to the \nameref{subsec:publisher_subscriber_interface}. It has however, no notion of a networked setup and could therefore be described as a two-directional message system between two destinations. With it, only setups with up to \textit{n} clients are possible. \subsubsection{OSC through PureData} \label{subsubsec:osc_through_puredata} To allow \gls{osc} communication, the \gls{ssr} incorporates a Lua based \gls{pd} external. It uses two externals (iemnet and pdlua) alongside a Lua library for parsing and creating \gls{xml} (SLAXML). \subsubsection{Sending and receiving} \label{subsubsec:sending_and_receiving} As mentioned in section~\nameref{subsec:publisher_subscriber_interface}, the NetworkSubscriber class (part of the \gls{ip} interface) implements the subscriber interface. This means: The network interface subscribes to the messages the publisher (the Controller instance) has to offer. Every time a function of the \gls{ssr}'s Controller instance, that was inherited from Publisher, is called, it will issue the call on all of its subscribers, too.\\ \cleardoublepage \subsection{Open Sound Control interface} \label{subsec:osc-interface} The networking interface conceived in the course of this work was developed in several branches, using the git version control system (written by Linus Torvalds, now maintained by Junio Hamano\footnote{\href{https://git-scm.com}{https://git-scm.com}}), publicly on Github\footnote{\href{https://github.com/dvzrv/ssr} {https://github.com/dvzrv/ssr}}. Internally the liblo library (further explained in~\ref{subsubsec:liblo}) was harnessed to implement \gls{osc} functionality (see~\ref{subsubsec:open-sound-control}) for the \gls{ssr}.\\ After initial conversations with the current maintainer Matthias Geier through the project's Github issue tracker\footnote{ \href{https://github.com/soundscaperenderer/ssr} {https://github.com/soundscaperenderer/ssr}}, different ideas were worked out to achieve a broad solution to the server-client and client-only setups and get a better understanding of the underlying design. Initial attempts, such as the mapping of a networking setup in the scene description\footnote{ \href{https://github.com/dvzrv/ssr/tree/distributed\_reproduction} {https://github.com/dvzrv/ssr/tree/distributed\_reproduction}} proved too restrictive though, as it would allow the networking functionality only to renderers, that use loudspeakers and mix scene description with networking description.\\ A nearly configuration-less approach, based on subscribing clients on sending poll messages to them proved more open (in the sense that it can be interfaced with by any \gls{osc}-capable application or programming language) and have less configuration overhead. With it, a diverse set of setups can be achieved (further described in~\ref{subsubsec:setups}), which at the same time remain dynamically configurable (using a plethora of \gls{osc} implementations) and debuggable using tests (further explored in~\ref{subsec:automated_tests}). \begin{figure}[!htb] \centering \includegraphics[scale=0.6, trim = 6mm 91mm 12mm 10mm, clip] {ssr-publisher-with-all-subscribers.pdf} \caption{A diagram depicting a simplified version of the \gls{pubsub} used within the \gls{ssr}.\\ {\color{pubsub-in}\textbf{--}} Calls from Publisher to Subscriber {\color{controller-in}\textbf{--}} Calls from Subscribers to Controller (Publisher) } \label{fig:ssr-publisher-with-all-subscribers} \end{figure} The main implementations of the interface, further described in the following subsections, can be found in the classes OscHandler (handling the \gls{osc} server), OscReceiver (handling incoming \gls{osc} messages and acting upon them in the context of the \gls{ssr} instance) and OscSender (responsible for reacting to calls from the \textbf{\nameref{subsec:publisher_subscriber_interface}} and sending of \gls{osc} messages to clients and server).\\ As shown in Figure~\ref{fig:ssr-publisher-with-all-subscribers}, the OscSender is another implementation of the Subscriber interface. This way, every call made through the Publisher (i.e.\ the Controller), will be made on the corresponding function in OscSender as well. With OscReceiver the \gls{osc} interface has direct access to the Controller and can make calls to it, on receiving a message. In its implementation approach the \gls{osc} interface therefore follows that of the \textbf{\nameref{subsec:ip-interface}}, expanding however in creating a client-server architecture, controlled by \textbf{\nameref{subsubsec:message_levels}}, using a unified \textbf{\nameref{subsubsec:message_interface}}.\\ \gls{ssr} client instances only evaluate messages of server instances they are subscribed to. Server instances only evaluate messages of client instances, that are subscribed to them. \subsubsection{Open Sound Control} \label{subsubsec:open-sound-control} \gls{osc} is an “open, transport-independent, message-based protocol developed for communication among computers, sound synthesizers, and other multimedia devices” \citep{website:oscv1.0} developed at the \gls{cnmat}. Its 1.0 specification was published by Matthew Wright in 2002 \citep{website:oscv1.0} and the protocol has found widespread implementations (as libraries) in several programming languages and through that many use-cases in free and closed audio and video related applications (e.g. Ardour \citep{website:ardour}, Cubase \citep{website:steinberg}, Max/MSP \citep{website:cycling74}, SuperCollider \citep{website:supercollider}) since then.\\ \gls{osc}'s syntax is defined by several parts, which are discussed briefly in this section.\\ \begin{itemize} \item Atomic data types, which are also reflected in type tags (see Table~\ref{tab:ssr-osc-data-type-acronyms} for details) \item Address patterns (a \gls{osc}-string starting with a “/”) \item Type tag string (a string, beginning with a “,”, holding a set of type tags, describing a collection of atomic data types) \item Arguments, a set of binary representations of each argument \item Messages, consisting (in sequence) of an address pattern, a type tag string and \textit{n} \gls{osc} arguments. \item Bundles, consisting of a set of Messages. \item Packets, the unit of transmission (sent over \gls{udp} or \gls{tcp}), consisting of a message or a bundle. \end{itemize} According to the specification, applications sending \gls{osc} packets are considered a client and the ones receiving packets a server. Applications can therefore be client and server at the same time.\\ \begin{table}[!htb] \scriptsize \centering \rowcolors{2}{table-background-one}{table-background-two} \begin{tabular}{ p{2cm} | p{8cm} } \textbf{\gls{osc} type tag} & \textbf{Type} \\ \hline \texttt{i} & int32 \\ \texttt{f} & float32 \\ \texttt{s} & \gls{osc}-string \\ \texttt{b} & \gls{osc}-blob \\ \hline \hline \texttt{h} & 64 bit big-endian two's complement integer\\ \texttt{t} & \gls{osc}-timetag\\ \texttt{d} & 64 bit (“double”) IEEE 754 floating point number\\ \texttt{S} & Alternate type represented as an \gls{osc}-string (for example, for systems that differentiate “symbols” from “strings”)\\ \texttt{c} & an ascii character, sent as 32 bits\\ \texttt{r} & 32 bit RGBA color\\ \texttt{m} & 4 byte MIDI message. Bytes from MSB to LSB are: port id, status byte, data1, data2\\ \texttt{T} & True. No bytes are allocated in the argument data.\\ \texttt{F} & False. No bytes are allocated in the argument data.\\ \texttt{N} & Nil. No bytes are allocated in the argument data.\\ \texttt{I} & Infinitum. No bytes are allocated in the argument data.\\ \texttt{[} & Indicates the beginning of an array. The tags following are for data in the Array until a close brace tag is reached.\\ \texttt{]} & Indicates the end of an array.\\ \end{tabular} \caption{Acronyms (type tags) for atomic data types, used in \gls{osc} messages and bundles \citep{website:oscv1.0}.}\\ The first four types define the standard \gls{osc} type tags, which should be understood by all implementations. The remaining are non-standard types, that are implemented by most (e.g. \nameref{subsubsec:liblo} implements all but array and RGBA color type). \label{tab:ssr-osc-data-type-acronyms} \end{table} As shown in Table~\ref{tab:ssr-osc-data-type-acronyms}, only int32, float32, \gls{osc}-string and \gls{osc}-blob are considered standardized. Most of the remaining non-standard types are however implemented and used by many different clients. For implementing the \gls{ssr} \gls{osc} interface, described in the following subsubsections, it was necessary to use the non-standard types \textit{True} and \textit{False} alongside the standard-types. \subsubsection{liblo} \label{subsubsec:liblo} Liblo \citep{website:liblo2017} is an implementation of the \gls{osc} protocol for \gls{posix} systems. It was initially developed by Steve Harris and is now actively maintained by Stephen Sinclair.\\ The library, written in C, offers a C++ abstraction layer and is released under the \gls{lgpl} v2.1 or greater. Additionally there are wrappers for the Perl and Python programming languages.\\ Due to its long standing availability and usage in many small and large-scale software projects, alongside its fairly straight forward implementability, it was chosen as the candidate for establishing a \gls{osc} interface for the \gls{ssr}.\\ At the time of writing liblo's lastet stable release (0.28) was issued on 27th January 2014. Many changes and improvements have been applied to the codebase since then. One of them is the implementation of a ServerThread for the C++ abstraction layer, which runs a Server instance on a separate thread automatically.\\ In programming, threads are a way to implement simultaneous and/ or asynchroneous execution of code. The liblo Server class, at the core of the C++ side of the library, is responsible for assigning a network port to listen to for incoming messages, listening for messages, executing code on their arrival (callback handling) and sending messages to clients. As most applications, facilitating liblo, use \gls{osc} only as a messaging system, it usually means, that the application itself is not single-purpose and is busy computing something else most of the time. Therefore it makes sense to run a Server instance on a separate background thread, to not interfere with the executional flow of the rest of the program.\\ The ServerThread class is able to free its ressources upon going ot of scope, known as \gls{raii}. For this reason, the latest development version, instead of the current stable version of liblo was chosen for the implementation. \subsubsection{Starting the SSR} \label{subsubsec:starting-the-ssr} The \gls{ssr} can be started with a rendering engine preselected (an executable postfixed by the supported rendering algorithm is provided by the software - e.g. \textbf{ssr-wfs}), or by selecting one through the configuration file, when using the standard executable named \textbf{ssr}. This way \gls{aap}, \gls{bs}, \gls{brs}, generic, \gls{nfc-hoa}, \gls{vbap}, \gls{wfs} renderers become available.\\ Additional features can be activated with the help of several flags to the executables. The customized ones, belonging to the \gls{osc} interface will be discussed in the following paragraphs. More information on the interplay between \gls{osc} messaging and the \nameref{subsec:publisher_subscriber_interface} can be found in \textbf{\nameref{subsubsec:message_interface}}. \paragraph{Client instance} \label{para:client-instance} By default the \gls{ssr} is started as an \gls{osc} client on port 50001\pdfcomment[color=red,icon=Note]{Handle failure to startup the liblo ServerThread more gracefully. Instead of segfaulting in case of port in use, throw error message and exit.}. As shown in Listing~\ref{lst:ssr-binaural-client-start}, it is possible to use a different port, by defining it with the help of the \textbf{-p} flag.\\ \begin{listing}[!htb] \begin{mdframed} \begin{minted}[fontsize=\footnotesize]{shell} ssr-binaural -p “50002” \end{minted} \end{mdframed} \caption{Starting the \gls{ssr} using the \gls{bs} renderer as an \gls{osc} client (default) on the non-standard port 50002.} \label{lst:ssr-binaural-client-start} \end{listing} Once started, the client instance waits to receive a poll message from a server instance (or an application, mimicking one), upon which it will subscribe to it. Only then it is possible for the server application to control the client instance to the full extent via \gls{osc}. \paragraph{Server instance} \label{para:server-instance} With the help of the \textbf{-N} flag \pdfcomment[color=red,icon=Note]{Make -N start server, client is default anyways}, it is possible to start the \gls{ssr} as an \gls{osc} server. In Listing~\ref{lst:ssr-binaural-server-start} additionally flag \textbf{-C} is used to specify an initial client \gls{ip} and its port (the flag actually accepts a comma-separated list of \gls{ip}-port pairs).\\ The \textbf{-p} flag, for setting a specific port is also available, when starting a server instance. \begin{listing}[!htb] \begin{mdframed} \begin{minted}[fontsize=\footnotesize]{shell} ssr-aap -N “server” -C “127.0.0.1:50002” \end{minted} \end{mdframed} \caption{Starting the \gls{ssr} using the \gls{aap} renderer as an \gls{osc} server, with an initial client on localhost, port 50002 provided. } \label{lst:ssr-binaural-server-start} \end{listing} When the server instance starts, it instantly starts sending out poll messages to all of its active clients. Clients provided by the \textbf{-C} flag are considered instantly active.\\ Additionally it's possible for clients (\gls{ssr} client instances, or \gls{osc} capable applications) to subscribe to the server instance, or be subscribed to it by another client, using a message level system further explained in~\ref{subsubsec:message_levels}. Every valid \gls{osc} message sent to the server instance will be delegated to all of its clients upon evaluation, again according to the aforementioned message level system. \paragraph{Verbosity} \label{para:verbosity} The \gls{ssr} can be started with several levels of verbosity. These are accessed by using the flag \textbf{-v}, up to three times (i.e. \textbf{-vvv}).\\ The higher the level of verbosity, the more messages will be printed by the application. This especially applies to the \gls{osc} interface part of the \gls{ssr}, as most incoming and outgoing messages will be printed to \gls{stdout} at a level of \textbf{-vv}. At a level of \textbf{-vvv}, additionally all incoming and outgoing messages, that are issued in very short intervals per default (see~\ref{subsubsec:message_levels} for details) will be printed. \subsubsection{Setups} \label{subsubsec:setups} The \gls{ssr} offers the possibility for many different \gls{osc} enabled client-server and client-only setups. They will be explained in the following paragraphs.\\ All examples provide audio input via a \gls{jack} client, which can be local (on each client's or server's host computer) or provided through external audio inputs from another host computer (e.g.\ through \gls{adat} or \gls{madi}). This however is not mandatory, as the \gls{ssr} is capable of playing back audio files directly.\\ The differences between server and client messaging is further elaborated upon in \textbf{\nameref{subsubsec:message_interface}}.\\ A special networked setup, in which the server instance is not rendering any audio, is discussed in \textbf{\nameref{subsec:implementing_a_nullrenderer}}. \paragraph{Client-Server, shared rendering} \label{para:client_server_shared_rendering} In Figure~\ref{fig:ssr-client-server-shared-output}, the setup shows \textit{1} to \textit{n} client instances, controlled by a server instance. All instances are receiving audio from an external \gls{jack} client or from reading local files. Collectively, the \textit{n} clients and the server are rendering audio on a shared output system (e.g. \gls{wfs} or \gls{hoa}).\\ The server instance is controlled through its \gls{gui}, sends out \gls{osc} messages to all \textit{n} clients and receives their updated information (again through \gls{osc}). \begin{figure}[!htb] \centering \includegraphics[scale=1.0, trim = 20mm 204mm 10mm 10mm, clip] {ssr-client-server-shared-output.pdf} \caption{A diagram displaying a \gls{ssr} client/server setup, in which the server and the clients render audio collectively (e.g. \gls{wfs}). The server instance is not controlled via \gls{osc}, but controls its clients through it.\\ {\color{osc-in}\textbf{--}} \gls{osc} input {\color{osc-out}\textbf{--}} \gls{osc} output {\color{audio-in}\textbf{--}} Audio input {\color{audio-out}\textbf{--}} Audio output } \label{fig:ssr-client-server-shared-output} \end{figure} \begin{figure}[!htb] \centering \includegraphics[scale=1.0, trim = 20mm 204mm 10mm 10mm, clip] {ssr-external-client-server-shared-output.pdf} \caption{A diagram displaying a \gls{ssr} client/server setup, in which the server and the clients render audio collectively (e.g. \gls{wfs}). The server instance is controlled by an \gls{osc} capable application (acting as another client) and controls its clients through \gls{osc} as well.\\ {\color{osc-in}\textbf{--}} \gls{osc} input {\color{osc-out}\textbf{--}} \gls{osc} output {\color{audio-in}\textbf{--}} Audio input {\color{audio-out}\textbf{--}} Audio output } \label{fig:ssr-external-client-server-shared-output} \end{figure} The setup shown in Figure~\ref{fig:ssr-external-client-server-shared-output} is similar to the previous one, with the exception, that the server instance is controlled by an external \gls{osc} capable application. This way, the server instance can also be run headless (without a \gls{gui}).\\ The set of \textit{n} clients report back to the server instance, which in turn reports back to the \gls{osc} enabled application (acting as another client). \cleardoublepage \paragraph{Client-Server, separate rendering} \label{para:client_server_separate_rendering} As shown in Figure~\ref{fig:ssr-client-server-separate-output}, it is possible to have a setup, in which, similar to the one described in Figure~\ref{fig:ssr-client-server-shared-output}, server and \textit{n} clients render the same sources, but on separate output systems (e.g.\ several \gls{bs}/\gls{brs} renderers or even a mixture of a \gls{wfs}/\gls{hoa} system and several \gls{bs}/\gls{brs} renderers).\\ \begin{figure}[!htb] \centering \includegraphics[scale=1.0, trim = 20mm 204mm 10mm 10mm, clip] {ssr-client-server-separate-output.pdf} \caption{A diagram displaying a \gls{ssr} client/server setup, in which the server and the clients render audio to separate outputs (e.g.\ multiple \gls{bs} renderers). The server instance is not controlled via \gls{osc}, but controls its clients through it.\\ {\color{osc-in}\textbf{--}} \gls{osc} input {\color{osc-out}\textbf{--}} \gls{osc} output {\color{audio-in}\textbf{--}} Audio input {\color{audio-out}\textbf{--}} Audio output } \label{fig:ssr-client-server-separate-output} \end{figure} Figure~\ref{fig:ssr-external-client-server-separate-output} is an example of a similar setup, but again using an external \gls{osc} capable application to control the server instance.\\ \begin{figure}[!htb] \centering \includegraphics[scale=1.0, trim = 20mm 204mm 10mm 10mm, clip] {ssr-external-client-server-separate-output.pdf} \caption{A diagram displaying a \gls{ssr} client/server setup, in which the server and the clients render audio separately (e.g.\ multiple \gls{bs} renderers). The server instance is controlled by an \gls{osc} capable application (acting as another client) and controls its clients through \gls{osc} as well.\\ {\color{osc-in}\textbf{--}} \gls{osc} input {\color{osc-out}\textbf{--}} \gls{osc} output {\color{audio-in}\textbf{--}} Audio input {\color{audio-out}\textbf{--}} Audio output } \label{fig:ssr-external-client-server-separate-output} \end{figure} \cleardoublepage \paragraph{Clients only} \label{para:clients_only} Using the new \gls{osc} interface, it is also possible to have client-only setups, in which an \gls{osc} capable application mimics a \gls{ssr} server. This way a set of \textit{n} clients can collectively (see Figure~\ref{fig:ssr-external-clients-only-shared-output}) or separately (see Figure~\ref{fig:ssr-external-clients-only-separate-output}) render audio, without the specific need of a \gls{ssr} server instance controlling them. The clients send their update information back to the controlling application.\\ Much of the functionality implemented in the server-side of the \gls{osc} interface however has to be reapplied to the controlling software and its behavior, when dealing with \gls{ssr} clients. \begin{figure}[!htb] \centering \includegraphics[scale=1.0, trim = 20mm 204mm 10mm 10mm, clip] {ssr-external-clients-only-shared-output.pdf} \caption{A diagram displaying a \gls{ssr} client cluster setup, in which a set of clients render audio collectively (e.g.\ medium or large-scale \gls{wfs} setup). An \gls{osc} capable application acts as a \gls{ssr} server instance and controls the clients.\\ {\color{osc-in}\textbf{--}} \gls{osc} input {\color{osc-out}\textbf{--}} \gls{osc} output {\color{audio-in}\textbf{--}} Audio input {\color{audio-out}\textbf{--}} Audio output } \label{fig:ssr-external-clients-only-shared-output} \end{figure} \begin{figure}[!htb] \centering \includegraphics[scale=1.0, trim = 20mm 204mm 10mm 10mm, clip] {ssr-external-clients-only-separate-output.pdf} \caption{A diagram displaying a \gls{ssr} client cluster setup, in which a set of clients render audio separately (e.g.\ multiple \gls{bs} renderers). An \gls{osc} capable application acts as a \gls{ssr} server instance and controls the clients.\\ {\color{osc-in}\textbf{--}} \gls{osc} input {\color{osc-out}\textbf{--}} \gls{osc} output {\color{audio-in}\textbf{--}} Audio input {\color{audio-out}\textbf{--}} Audio output } \label{fig:ssr-external-clients-only-separate-output} \end{figure} \cleardoublepage \subsubsection{Message levels} \label{subsubsec:message_levels} To be able to distinguish between types of clients and servers, several message levels were implemented for the \gls{osc} interface conceived in the course of this work.\\ The \textit{enumeration class} \textit{MessageLevel} \pdfcomment[color=red,icon=Note]{Don't allow static\_casts above MessageLevel::GUI\_SERVER} (see Listing~\ref{lst:ssr_global.h}) defines the four types \textit{CLIENT}, \textit{GUI\_CLIENT}, \textit{SERVER}, \textit{GUI\_SERVER}, which are represented as non-negative integers (in ascending order), starting from 0.\\ \begin{listing}[!htb] \begin{mdframed} \inputminted[numbers=left, firstline=54, lastline=60, fontsize=\footnotesize]{c++}{../../ssr/src/ssr_global.h} \end{mdframed} \caption{src/ssr\_global.h: enum class MessageLevel} \label{lst:ssr_global.h} \end{listing}\\ \gls{ssr} client instances subscribe to (\gls{ssr}) server instances with the \textit{MessageLevel} \textit{CLIENT} by default. Server instances get the \textit{MessageLevel} \textit{SERVER} assigned to by each client on subscribing to it.\\ In the \gls{osc} interface it is implemented as follows: A (recycable and reconfigurable) list of clients is held by a server instance, which enables for the \textit{MessageLevel} to change for each client. Every client instance holds a (reconfigurable) server representation, that enables for the \textit{MessageLevel} to change for each client towards its server.\\ Several messages, such as information related to \gls{cpu} load or master signal level are not useful for a rendering client (additionally they are issued in very short intervals, which can lead to performance issues), which is why they are only sent to clients with a \textit{MessageLevel} of \textit{GUI\_CLIENT} or servers with a \textit{MessageLevel} of \textit{GUI\_SERVER}.\\ Lightweight \gls{osc} capable applications used to control a \gls{ssr} server instance are clients to said server instance. An elevated \textit{MessageLevel} of \textit{SERVER} (instead of \textit{CLIENT}) enables them to send messages to the server and have them evaluated. Analogous to a server instance holding a \textit{MessageLevel} of \textit{GUI\_SERVER} towards its clients, a client instance can hold the same \textit{MessageLevel} towards a server instance to receive the above mentioned performance heavy \gls{osc} messages.\\ How the setting up of message levels is achieved is further elaborated upon in the following section. \subsubsection{Message interface} \label{subsubsec:message_interface} \gls{osc} offers the possibility of a hierarchical path tree, that can be used to group messages by type (i.e.\ context). In conjunction with messages only understood by client or server (or a context dependant meaning), most of the messages understood by the \nameref{subsec:ip-interface} are implemented. Additional features, related to server-client and client-only functionality were integrated as well.\\ Generally it can be distinguished between \textit{direct} messages - sent from a server (or an application mimicking one) to a client to trigger processing (see Table~\ref{tab:ssr-osc-processing-tracker-transport}), reference (see Table~\ref{tab:ssr-osc-reference}), scene (see Table~\ref{tab:ssr-osc-scene}), source (see Table~\ref{tab:ssr-osc-source}), tracker (see Table~\ref{tab:ssr-osc-processing-tracker-transport}) or transport (see Table~\ref{tab:ssr-osc-processing-tracker-transport}) related operations in the \gls{ssr} - and \textit{update} messages (see Table~\ref{tab:ssr-osc-update}) - sent from a client to a server upon successful processing an operation related to a \textit{direct} message.\\ A special set of \textit{direct} message are the subscribe and message level (see Table~\ref{tab:ssr-osc-subscribe}) and poll and message level (see Table~\ref{tab:ssr-osc-client-poll-message-level}) messages. The former - understood only by \gls{ssr} server instances - enable clients to subscribe (with a certain message level) or subscribe other clients (with a predefined message level) and set their own message level or that of another client. The latter set - only understood by clients - enables servers (or applications mimicking one) to poll yet unsubscribed clients to have them subscribe and subscribed clients to reply with an alive message. Similar to the message level message understood by server instances, the one understood by clients sets the message level (of the server representation in the client).\\ When starting a \gls{ssr} server instance (see~\ref{para:server-instance}), it responds to the messages shown in Table~\ref{tab:ssr-osc-subscribe} ,~\ref{tab:ssr-osc-scene} ,~\ref{tab:ssr-osc-processing-tracker-transport} ,~\ref{tab:ssr-osc-reference} and~\ref{tab:ssr-osc-update}. \begin{table}[!htb] \scriptsize \centering \rowcolors{2}{table-background-one}{table-background-two} \begin{tabular}{ p{2cm} | p{1cm} | p{3.5cm} | p{3cm} } \textbf{Path} & \textbf{Types} & \textbf{Description} & \textbf{Example}\\ \hline \texttt{/message\_level} & i & Set message level of sender & \texttt{[/message\_level, 1]} \\ \texttt{/message\_level} & ssi & Set message level of a specific client & \texttt{[/message\_level, “127.0.0.1”, “50002”, 1]} \\ \texttt{/subscribe} & F & Unsubscribe sender & \texttt{[/subscribe, false]} \\ \texttt{/subscribe} & Fss & Unsubscribe specific client & \texttt{[/subscribe, false, “127.0.0.1”, “50002”]}\\ \texttt{/subscribe} & T & Subscribe sender & \texttt{[/subscribe, true]} \\ \texttt{/subscribe} & Ti & Subscribe sender with specific message level & \texttt{[/subscribe, true, 1]} \\ \texttt{/subscribe} & Tssi & Subscribe specific client with specific message level & \texttt{[/subscribe, true, “127.0.0.1”, “50002”, 1]} \\ \end{tabular} \caption{\gls{osc} messages relevant for subscribing and setting of message levels for clients.\\ } \caption*{Understood by server.\\ Data types and their acronyms are listed in Table~\ref{tab:ssr-osc-data-type-acronyms}. } \label{tab:ssr-osc-subscribe} \end{table} \begin{table}[!htb] \scriptsize \centering \rowcolors{2}{table-background-one}{table-background-two} \begin{tabular}{ p{2cm} | p{1cm} | p{3.5cm} | p{3cm} } \textbf{Path} & \textbf{Types} & \textbf{Description} & \textbf{Example}\\ \hline \texttt{/message\_level} & i & Set message level of sender (the server) & \texttt{[/message\_level, 1]} \\ \texttt{/poll} & & Poll client (continously sent), triggering subscribe (see Table~\ref{tab:ssr-osc-subscribe}) & \texttt{[/poll]} \\ \end{tabular} \caption{\gls{osc} messages relevant for polling and setting of message levels for servers subscribed to.\\ } \caption*{Understood by client.\\ Data types and their acronyms are listed in Table~\ref{tab:ssr-osc-data-type-acronyms}. } \label{tab:ssr-osc-client-poll-message-level} \end{table} \begin{table}[!htb] \scriptsize \centering \rowcolors{2}{table-background-one}{table-background-two} \begin{tabular}{ p{3cm} | p{1.2cm} | p{3.5cm} | p{3cm} } \textbf{Path} & \textbf{Types} & \textbf{Description} & \textbf{Example}\\ \hline \texttt{/processing/state} & F & Unset processing state & \texttt{[/processing/state, false]} \\ \texttt{/processing/state} & T & Set processing state & \texttt{[/processing/state, true]} \\ \texttt{/tracker/reset} & & Reset tracker & \texttt{[/tracker/reset]} \\ \texttt{/transport/rewind} & & Rewind the \gls{jack} transport & \texttt{[/transport/rewind]} \\ \texttt{/transport/seek} & s & Seek to time code in \gls{jack} transport & \texttt{[/transport/seek, “42:00:00”]} \\ \texttt{/transport/state} & F & Unset \gls{jack} transport state & \texttt{[/transport/state, false]} \\ \texttt{/transport/state} & T & Set \gls{jack} transport state & \texttt{[/transport/state, true]} \\ \end{tabular} \caption{\gls{osc} messages relevant for processing, tracker and (\gls{jack}) transport related settings.\\ } \caption*{Understood by server and client.\\ Data types and their acronyms are listed in Table~\ref{tab:ssr-osc-data-type-acronyms}. } \label{tab:ssr-osc-processing-tracker-transport} \end{table} \begin{table}[!htb] \scriptsize \centering \rowcolors{2}{table-background-one}{table-background-two} \begin{tabular}{ p{4.3cm} | p{1cm} | p{2.5cm} | p{4.4cm} } \textbf{Path} & \textbf{Types} & \textbf{Description} & \textbf{Example}\\ \hline \texttt{/reference/orientation} & f & Set azimuth of reference point & \texttt{[/reference/orientation, -90.0]} \\ \texttt{/reference/position} & ff & Set position of reference & \texttt{[/reference/position, 1.5, 2.0]} \\ \texttt{/reference\_offset/orientation} & f & Set azimuth of reference offset position & \texttt{[/reference\_offset/orientation, -90.0]} \\ \texttt{/reference\_offset/position} & ff & Set position of reference offset & \texttt{[/reference\_offset/position, 1.5, 2.0]}\\ \end{tabular} \caption{\gls{osc} messages relevant for reference management.\\ } \caption*{Understood by server and client.\\ Data types and their acronyms are listed in Table~\ref{tab:ssr-osc-data-type-acronyms}. } \label{tab:ssr-osc-reference} \end{table} \begin{table}[!htb] \scriptsize \centering \rowcolors{2}{table-background-one}{table-background-two} \begin{tabular}{ p{4.5cm} | p{0.9cm} | p{2.5cm} | p{4.3cm} } \textbf{Path} & \textbf{Types} & \textbf{Description} & \textbf{Example} \\ \hline \texttt{/scene/amplitude\_reference \_distance} & f & Set amplitude reference distance. & \texttt{[/scene/amplitude\_reference \_distance, 6.0]}\\ \texttt{/scene/auto\_rotate\_sources} & F & Disable automatic rotation of sources. & \texttt{[/scene/auto\_rotate\_sources, false]}\\ \texttt{/scene/auto\_rotate\_sources} & T & Enable automatic rotation of sources. & \texttt{[/scene/auto\_rotate\_sources, true]}\\ \texttt{/scene/clear} & & Delete all sources & \texttt{[/scene/clear]}\\ \texttt{/scene/decay\_exponent} & f & Set amplitude decay exponent in virtual space ($1/r^{exp}$). & \texttt{[/scene/decay\_exponent, 2.0]}\\ \texttt{/scene/load} & s & Load scene from \gls{asdf} file. & \texttt{[/scene/load, “example.asd”]}\\ \texttt{/scene/master\_signal\_level} & f & Set the renderers signal level. & \texttt{[/scene/master\_signal\_level, -20]}\\ \texttt{/scene/save} & s & Save scene to \gls{asdf} file. & \texttt{[/scene/save, “example.asd”]}\\ \texttt{/scene/volume} & f & Set scene master volume. & \texttt{[/scene/volume, 0.23]}\\ \end{tabular} \caption{\gls{osc} messages relevant for scene management.\\ } \caption*{Understood by server and client.\\ Data types and their acronyms are listed in Table~\ref{tab:ssr-osc-data-type-acronyms}. } \label{tab:ssr-osc-scene} \end{table} \begin{table}[!htb] \scriptsize \centering \rowcolors{2}{table-background-one}{table-background-two} \begin{tabular}{ p{3.5cm} | p{1.5cm} | p{2.5cm} | p{4.3cm} } \textbf{Path} & \textbf{Types} & \textbf{Description} & \textbf{Example} \\ \hline \texttt{/source/delete} & i & Delete source with given id & \texttt{[/source/delete, 1]}\\ \texttt{/source/file\_channel} & ii & Set a source's file channel & \texttt{[/source/file\_channel, 1, 2]}\\ \texttt{/source/file\_name\_ or\_port\_number} & is & Set a source's file name or port number & \texttt{[/source/file\_name\_ or\_port\_number, 1, “1”]}\\ \texttt{/source/port\_name } & is & Set a source's \gls{jack} input port name & \texttt{[/source/port\_name, 1, “system:capture\_2”]}\\ \texttt{/source/gain} & if & Set a source's gain & \texttt{[/source/gain, 1, 0.2]}\\ \texttt{/source/model} & is & Set a source's model & \texttt{[/source/model, 1, “point”]}\\ \texttt{/source/mute} & iF & Unmute a source & \texttt{[/source/mute, 1, false]}\\ \texttt{/source/mute} & iT & Mute a source & \texttt{[/source/mute, 1, true]}\\ \texttt{/source/name} & is & Set a source's name & \texttt{[/source/name, 1, “Daisy”]}\\ \texttt{/source/new} & i & Create a new source stub using id & \texttt{[/source/new, 1]}\\ \texttt{/source/new} & sssffffTFF & Create a new source (auto-generated id) with name, model, port number, X-coordinate, Y-coordinate, orientation, gain, movability, orientation movability and mute status & \texttt{[/source/new, “Daisy”, “point”, “1”, 1.0, 2.5, 90.0, 0.2, true, false, false]} \\ \texttt{/source/new} & sssffffisTFF & Create a new source (auto-generated id) with name, model, port number, X-coordinate, Y-coordinate, orientation, gain, file channel, properties file, movability, orientation movability and mute status & \texttt{[/source/new, “Daisy”, “point”, “1”, 1.0, 2.5, 90.0, 0.2, 2, “properties.xml”, true, false, false]} \\ \texttt{/source/orientation} & if & Set a source's orientation & \texttt{[/source/orientation, 1, -90.0]}\\ \texttt{/source/position} & iff & Set a source's position & \texttt{[/source/position, 1, 1.5, 2.0]}\\ \texttt{/source/position\_fixed} & iF & Set a source movable & \texttt{[/source/position\_fixed, 1, false]}\\ \texttt{/source/position\_fixed} & iT & Set a source immovable & \texttt{[/source/position\_fixed, 1, true]}\\ \texttt{/source/properties\_file} & is & Set a source's properties file & \texttt{[/source/properties\_file, 1, “source-properties.xml”]}\\ \end{tabular} \caption{\gls{osc} messages relevant for source management.\\ } \caption*{Understood by server and client.\\ Data types and their acronyms are listed in Table~\ref{tab:ssr-osc-data-type-acronyms}. } \label{tab:ssr-osc-source} \end{table} \begin{table}[!htb] \scriptsize \centering \rowcolors{2}{table-background-one}{table-background-two} \begin{tabular}{ p{6.5cm} | p{0.9cm} | p{4.3cm} } \textbf{Path} & \textbf{Types} & \textbf{Description} \\ \hline \texttt{/update/cpu\_load} & f & \gls{cpu} load changes.\\ \texttt{/update/processing/state} & T & Processing state is set.\\ \texttt{/update/processing/state} & F & Processing state is unset.\\ \texttt{/update/reference/orientation} & f & Reference orientation changes.\\ \texttt{/update/reference/position} & ff & Reference position changes.\\ \texttt{/update/reference\_offset/orientation} & f & Reference offset orientation changes.\\ \texttt{/update/reference\_offset/position} & ff & Reference offset position changes.\\ \texttt{/update/scene/amplitude\_reference\_distance} & f & Amplitude reference distance changes.\\ \texttt{/update/scene/auto\_rotate\_sources} & T & Auto rotation of sources is set.\\ \texttt{/update/scene/auto\_rotate\_sources} & F & Auto rotation of sources is unset.\\ \texttt{/update/scene/decay\_exponent} & f & The scene's decay exponent has changed.\\ \texttt{/update/scene/master\_signal\_level} & f & Master signal level has changed.\\ \texttt{/update/scene/sample\_rate} & i & Sample rate of the scene changed.\\ \texttt{/update/scene/volume} & f & Volume of the scene has changed.\\ \texttt{/update/source/delete} & i & A source with given id was deleted. \\ \texttt{/update/source/file\_channel} & ii & A source's file channel was set.\\ \texttt{/update/source/file\_name\_or\_port\_number} & is & A source's file name or port number was set.\\ \texttt{/update/source/gain} & if & A source's gain was set.\\ \texttt{/update/source/length} & ii & A source's length was set.\\ \texttt{/update/source/level} & if & A source's output level has changed.\\ \texttt{/update/source/model} & is & A source's model was set.\\ \texttt{/update/source/mute} & iF & A source was unmuted.\\ \texttt{/update/source/mute} & iT & A source was muted.\\ \texttt{/update/source/name} & is & A source's name was set.\\ \texttt{/update/source/orientation} & if & A source's orientation was set. \\ \texttt{/update/source/new} & i & A new source with given id was created. \\ \texttt{/update/source/port\_name} & is & A source's \gls{jack} port\_name was set. \\ \texttt{/update/source/position} & iff & A source's position was set. \\ \texttt{/update/source/position\_fixed} & iF & A source was set to be movable. \\ \texttt{/update/source/position\_fixed} & iT & A source was set to be immovable. \\ \texttt{/update/source/properties\_file} & is & A source's properties\_file was set.\\ \texttt{/update/transport/seek} & s & \gls{jack} transport seeked to a timecode position.\\ \texttt{/update/transport/state} & F & \gls{jack} transport was stopped.\\ \texttt{/update/transport/state} & T & \gls{jack} transport was started.\\ \end{tabular} \caption{\gls{osc} messages for updating information on \gls{cpu} load, processing, reference, scene, source, and transport of clients on a server.\\ No examples are given, as they are mostly analogous to the ones in Table~\ref{tab:ssr-osc-processing-tracker-transport} ,~\ref{tab:ssr-osc-scene} and~\ref{tab:ssr-osc-source}. } \caption*{Understood by server.\\ Data types and their acronyms are listed in Table~\ref{tab:ssr-osc-data-type-acronyms}. } \label{tab:ssr-osc-update} \end{table} \cleardoublepage \subsubsection{Workflow Examples} \label{subsubsec:workflow_examples} Using any \gls{osc} capable programming language or application enables for communication with the \gls{ssr}. The following examples illustrate simple workflows using \gls{sclang} and should therefore be \gls{os} agnostic.\\ \paragraph{Controlling a server} \label{para:controlling_a_server} \begin{listing}[!htb] \begin{mdframed} \inputminted[numbers=left, firstline=3, lastline=29, fontsize=\footnotesize]{supercollider}{../../ssr/supercollider/workflows.scd} \end{mdframed} \caption{supercollider/workflows.scd: \gls{sclang} as client controlling a \gls{ssr} server instance} \label{lst:ssr-workflow-sclang-controls-server} \end{listing}\\ As shown in Listing~\ref{lst:ssr-workflow-sclang-controls-server}, it is necessary to subscribe to the server instance with a \textit{MessageLevel} of \textit{SERVER} or higher.\\ After doing so, also all \textit{direct} \gls{osc} messages (i.e. Table~\ref{tab:ssr-osc-source},~\ref{tab:ssr-osc-scene} ,~\ref{tab:ssr-osc-reference},~\ref{tab:ssr-osc-subscribe}) are evaluated when sent to the \gls{ssr}.\\ The server instance will relay valid messages to all of its active clients. \paragraph{Server mimicry} \label{para:server_mimicry} \begin{listing}[!htb] \begin{mdframed} \inputminted[numbers=left, firstline=31, lastline=61, fontsize=\footnotesize]{supercollider}{../../ssr/supercollider/workflows.scd} \end{mdframed} \caption{supercollider/workflows.scd: \gls{sclang} mimics server, controlling a \gls{ssr} client instance} \label{lst:ssr-workflow-sclang-is-server-controls-client} \end{listing}\\ When mimicking a \gls{ssr} server instance in a client-only setup (e.g.\ Figure~\ref{fig:ssr-external-clients-only-shared-output} or Figure~\ref{fig:ssr-external-clients-only-separate-output}), it is necessary to send a poll message to the client instance to make it subscribe (which sets the server's address and port up internally).\\ Afterwards - similar to the example in the paragraph \nameref{para:clients_only} - all \textit{direct} \gls{osc} messages are accepted by the client instance, when coming from the server address and port.\\ An interesting concept here is to (temporarily) set a different \textit{MessageLevel} for the application acting as a server (e.g.\ to \textit{GUI\_SERVER}), to receive \gls{gui} relevant messages, as explained in \nameref{subsubsec:message_interface}.\\ \subsection{Automated tests} \label{subsec:automated_tests} The \gls{ssr} was developed without the help of a test framework, which is responsible to test its components, after they have been changed. This means, that internal (e.g.\ the \gls{pubsub} interface), or external (e.g.\ the \gls{ip} or \gls{osc} interface) functionality might or might not work as expected. To test the \gls{osc} interface's logical coherency and robustness automatically, a set of tests were written in \gls{sclang}.\\ The tests are divided into those probing robustness of the \gls{osc} interface and others probing its functionality. The robustness tests further divide into server and client specific tests, where authorized and unauthorized access is tried. The functionality tests are grouped by tests for general operability, i.e.\ testing certain features or workflows once and long-running tests, where features are tried repeatedly.\\ \subsubsection{Robustness} \label{subsubsec:robustness} Listing~\ref{lst:ssr-tests-sclang-unsubscribed-controls-server} and~\ref{lst:ssr-tests-sclang-subscribed-controls-server} describe server-side tests for robustness. While the first will not lead to any processed action by the server, the latter will. This is explained by \gls{sclang} not being a subscribed client with a \textit{MessageLevel} of \textit{SERVER} or higher in the first case. In the second test however \gls{sclang} subscribes to the \gls{ssr} server instance, which is why the \gls{osc} messages are evaluated in this case.\\ \begin{listing}[!htb] \begin{mdframed} \inputminted[numbers=left, firstline=145, lastline=158, fontsize=\footnotesize]{supercollider} {../../ssr/supercollider/tests.scd} \end{mdframed} \caption{supercollider/tests.scd: \gls{sclang} (unsubscribed) tries to control a \gls{ssr} server} \label{lst:ssr-tests-sclang-unsubscribed-controls-server} \end{listing}\\ \begin{listing}[!htb] \begin{mdframed} \inputminted[numbers=left, firstline=160, lastline=183, fontsize=\footnotesize]{supercollider} {../../ssr/supercollider/tests.scd} \end{mdframed} \caption{supercollider/tests.scd: \gls{sclang} (subscribed) tries to control a \gls{ssr} server} \label{lst:ssr-tests-sclang-subscribed-controls-server} \end{listing}\\ The tests described in Listing~\ref{lst:ssr-tests-sclang-controls-client-unpolled} and~\ref{lst:ssr-tests-sclang-controls-client-polled} are client-side tests for robustness, that work in a similar fashion to the aforementioned server-side tests. While in the first case, the sent \gls{osc} messages are not evaluated, because \gls{sclang}, mimicking a server instance (see~\ref{para:server_mimicry}), did not poll the \gls{ssr} client instance up front, in the second case the messages are evaluated, because it did poll the client first. \begin{listing}[!htb] \begin{mdframed} \inputminted[numbers=left, firstline=185, lastline=197, fontsize=\footnotesize]{supercollider} {../../ssr/supercollider/tests.scd} \end{mdframed} \caption{supercollider/tests.scd: \gls{sclang} tries to control a \gls{ssr} client (without polling it)} \label{lst:ssr-tests-sclang-controls-client-unpolled} \end{listing}\\ \begin{listing}[!htb] \begin{mdframed} \inputminted[numbers=left, firstline=199, lastline=214, fontsize=\footnotesize]{supercollider} {../../ssr/supercollider/tests.scd} \end{mdframed} \caption{supercollider/tests.scd: \gls{sclang} tries to control a \gls{ssr} client (with previously polling it)} \label{lst:ssr-tests-sclang-controls-client-polled} \end{listing}\\ In all tests for robustness, the attempt is made to break the implementation of the \nameref{subsubsec:message_interface}. This is achieved by purposely using ranges of data types for messages, that are not allowed, or not defined in the \gls{ssr}'s internal implementation.\\ Two examples for weak spot exploitations are the use of negative integers for \glspl{id} in source related messages (only non-zero, non-negative \glspl{id} are allowed internally) or supplying an empty string as hostname or port number for subscription messages.\\ The first example will lead to undefined behavior, if the range is not checked in the implementation, as a \textit{static\_cast} is used internally to cast the value of the message data type (\textit{unsigned int}) to the one expected by the \gls{ssr}'s Controller implementation (\textit{signed int}) and the outcome of said operation is implementation dependant (depending on the \gls{os} in use).\\ The second example, if not checked for empty string, would lead to the \gls{osc} interface trying to create a possibly defective address and send poll messages out to it.\\ While only some of the above mentioned scenarios could lead to a crash of the program (under certain circumstances), unhandled all of them waste ressources, which is undesired. To circumvent possibly harmful input using the \gls{osc} interface, a set of sanity checks were implemented, that only allow for a received message to be processed, if all of its components fit the requirements. \subsubsection{Functionality and operability} \label{subsubsec:functionality_and_operability} \begin{listing}[!htb] \begin{mdframed} \inputminted[numbers=left, firstline=216, lastline=225, fontsize=\footnotesize]{supercollider} {../../ssr/supercollider/tests.scd} \end{mdframed} \caption{supercollider/tests.scd: \gls{sclang} controls a \gls{ssr} client (with previously polling it), creating several sources and moving them} \label{lst:ssr-tests-sclang-sources} \end{listing}\\ The test described in Listing~\ref{lst:ssr-tests-sclang-sources} is a test for functionality, which also serves as a long-running stress test for the \gls{ssr}. It creates 20 sources, that are then moved around randomly, every 100ms, for 100 seconds, which on a Lenovo W540, with an Intel i7-4700MQ and 16Gb RAM created less than 50\% of \gls{cpu} load.\\ Based on the above mentioned tests, the basic functionality of the \gls{osc} interface can be guaranteed and depending on the host's hardware also a maximum degree of capacity utilization can be estimated, when observing the \gls{ssr}'s workload towards the system, while using the long-running tests.\\ It has to be mentioned, that a higher load can be observed, when using higher levels of verbosity (especially above \textit{-vv}). This is explained by the fact, that the \gls{ssr} will print out every \gls{osc} message received and sent above the aforementioned verbosity level. \cleardoublepage \section{Discussion} \label{sec:discussion} \cleardoublepage \subsection{Implementing a NullRenderer} \label{subsec:implementing_a_nullrenderer} \begin{figure}[!htb] \centering \includegraphics[scale=1.0, trim = 20mm 204mm 10mm 10mm, clip] {ssr-client-server-clients-only-shared-output.pdf} \caption{A diagram displaying a \gls{ssr} client/server setup, in which only the clients render audio collectively (e.g.\ medium or large-scale \gls{wfs}). The server instance is not controlled via \gls{osc}, but controls its clients through it. Additionally its rendering engine does not have any outputs.\\ {\color{osc-in}\textbf{--}} \gls{osc} input {\color{osc-out}\textbf{--}} \gls{osc} output {\color{audio-in}\textbf{--}} Audio input {\color{audio-out}\textbf{--}} Audio output } \label{fig:ssr-client-server-clients-only-shared-output} \end{figure} \begin{figure}[!htb] \centering \includegraphics[scale=1.0, trim = 20mm 204mm 10mm 10mm, clip] {ssr-client-server-clients-only-separate-output.pdf} \caption{A diagram displaying a \gls{ssr} client/server setup, in which only the clients render audio to separate outputs (e.g.\ multiple \glspl{bs} renderers). The server instance is not controlled via \gls{osc}, but controls its clients through it. Additionally its rendering engine does not have any outputs.\\ {\color{osc-in}\textbf{--}} \gls{osc} input {\color{osc-out}\textbf{--}} \gls{osc} output {\color{audio-in}\textbf{--}} Audio input {\color{audio-out}\textbf{--}} Audio output } \label{fig:ssr-client-server-clients-only-separate-output} \end{figure} \cleardoublepage \subsection{Implementing AlienLoudspeaker} \label{subsec:implementing_alienloudspeaker} \subsection{Status messages} \label{subsec:status_messages} \subsection{Assigning inputs on the fly} \label{subsec:assigning_inputs_on_the_fly} \subsection{Interpolation of moving sources} \label{subsec:interpolation_of_moving_sources} \pagestyle{empty} \cleardoublepage \appendix \cleardoublepage \bibliographystyle{plainnat} \bibliography{../bib/ssr-networking} \end{document}