\documentclass{article}

\usepackage{graphicx}
\usepackage{multirow}
\usepackage{fancyhdr}

\makeindex{}

\makeatletter
\textheight 625pt
\textwidth 460pt
\oddsidemargin 0pt
\evensidemargin 0pt
\marginparwidth 50pt
\topmargin 0pt
\brokenpenalty=10000

% Makes fancy headings and footers.
\pagestyle{fancy}

\fancyhf{} % Clears all fields.

\lhead{\scshape Bardeau et al., 2017}
\rhead{\scshape \nouppercase{\rightmark}}   %
\fancyfoot[C]{\bfseries \thepage{}}

\renewcommand{\headrulewidth}{0pt}    %
\renewcommand{\footrulewidth}{0pt}    %

\renewcommand{\sectionmark}[1]{%
  \markright{\thesection. \MakeLowercase{#1}}}
\renewcommand{\subsectionmark}[1]{}

% \fancypagestyle{firststyle}
% {
%    \fancyhf{}
%    \fancyhead[C]{Original version at \tt http://iram-institute.org/medias/uploads/class-herschel-fits.pdf}
%    \fancyfoot[C]{\bfseries \thepage{}}
% }


\newcommand{\ie} {{\em i.e.}}
\newcommand{\eg} {{\em e.g.}}
\newcommand{\cf} {cf.}

\newcommand{\gildas}    {\mbox {\bf GILDAS}}
\newcommand{\sic}       {\mbox {\bf SIC}}
\newcommand{\greg}      {\mbox {\bf GREG}}
\newcommand{\class}     {\mbox {\bf CLASS}}
\newcommand{\classic}   {\mbox {\bf CLASSIC}}

\newcommand{\aas} {{\tt Associated Arrays}}
\newcommand{\aaa} {{\tt Associated Array}}
\newcommand{\ry} {{\tt RY}}
\newcommand{\rx} {{\tt RX}}

%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%

\begin{document}

% \thispagestyle{firststyle}

\begin{center}
    \includegraphics[width=5cm]{gildas-logo}\\[2\bigskipamount]
  \huge{}
  IRAM Memo 2017-?\\
  Virtual buffering in \class\\
  DRAFT VERSION\\[3\bigskipamount]
  \large{}
  S. Bardeau$^1$, J. Pety$^{1,2}$\\[0.5cm]
  1. IRAM (Grenoble)\\
  2. Observatoire de Paris\\[\bigskipamount]
  \normalsize{}
  July, $26^{th}$ 2017\\
  Version 0.2
\end{center}

\begin{abstract}
  In this document we measure the benefits of the system's cache when
  accessing several times the same \class\ file under several
  conditions, and we compare them with the internal buffering scheme
  implemented in \class\ itself. We argue the pros and cons for
  the \class\ internal buffer and finally explain why it is removed.\\

  Keywords: \class\ Data Format

  Related documents: \class\ documentation
\end{abstract}

\newpage{}
\tableofcontents{}

\newpage{}

\section{Benchmarks}

\subsection{Strategy}

The benchmarks are performed with the 5 tests described in the
subsequent sections, accessing the data with different tools and/or
caching conditions. Except explicit mention, the system used is a
Scientific Linux 6.4 with 24 GB RAM, and programs are compiled with
gcc/gfortran 5.2.0.\\

The data set is composed of 2 \class{} files (\classic{} binary
container version 2) of comparable size on disk:
\begin{enumerate}

  \item WILMA data
  \begin{itemize}
    \item file size 4,196 MBytes,
    \item 1,552,157 spectra,
    \item 465 channels per spectrum,
    \item 795 words per entry (descriptor + header + data),
  \end{itemize}

  \item FTS data
  \begin{itemize}
    \item file size 4,608 MBytes,
    \item 57,666 spectra,
    \item 20,737 channels per spectrum,
    \item 20,840 words per entry (descriptor + header + data),
  \end{itemize}
\end{enumerate}

The system, user, and elapsed times presented in the next tables are
median times when running 10 times the tests under identical
conditions. For simplicity, the files are actually duplicated 10 times
and read once each (instead of reading 10 times the same file). Median
times are prefered instead of mean times because once in a while, an
outlier is seen in the data (e.g. the system is performing other
actions out of this context).

\begin{table}[htb]
\centering
\caption{Run descriptions, in particular regarding the system cache and
  \class{} virtual buffer status (YES/NO/PARTIAL) at the beginning of the
  run. \emph{small} (resp. \emph{large}) virtual buffer means that the
  observation headers and data fit partially (resp. entirely) in the
  virtual buffer. The virtual buffer is reset (emptied) at runs 9 and 13
  while the file is kept in the system cache.}
\vspace{2ex}
\begin{tabular}{l|c|ccc}
  \hline
  \hline
  Test & Run & In system cache & Virtual actived & In virtual \\
                        & \#  & Y/N & Y/N & Y/N/P \\
  \hline
  {\tt cat}             &  1  &  N  &     &     \\
                        &  2  &  Y  &     &     \\
  \hline
  {\tt classic} program &  3  &  N  &     &     \\
                        &  4  &  Y  &     &     \\
  \hline
  \class{} procedure,   &  5  &  N  &  N  &     \\
  virtual buffer off    &  6  &  Y  &  N  &     \\
  \hline
  \class{} procedure,   &  7  &  N  &  Y  &  N  \\
  small virtual buffer  &  8  &  Y  &  Y  &  P  \\
                        &  9  &  Y  &  Y  &  N  \\
                        & 10  &  Y  &  Y  &  P  \\
  \hline
  \class{} procedure,   & 11  &  N  &  Y  &  N  \\
  large virtual buffer  & 12  &  Y  &  Y  &  Y  \\
                        & 13  &  Y  &  Y  &  N  \\
                        & 14  &  Y  &  Y  &  Y  \\
  \hline
\end{tabular}
\label{tab:descr}
\end{table}

\subsection{Best system performances}

We run here the simplest possible test in order the measure the system
capabilites. We use the system command {\tt cat} which reads
sequentially all the file without interpreting the bytes. The output
is redirected to {\tt /dev/null}. One may also think in a small
dedicated C program, optimized \eg\ for best sector size. The results
are shown in the Tables~\ref{tab:wilma-classicv2} and
\ref{tab:fts-classicv2}, runs 1 and 2. As expected, reading the file
is extremely fast when it has been saved in the system cache (about 50
times faster).

\begin{table}[p]
\centering
\caption{System, user, and elapsed times for reading the WILMA data file
  with the various tests described in Table~\ref{tab:descr}. In the
  column Elapsed [\%], the reference (100\%) is taken when reading the
  file the first time with a \class{} procedure with the virtual buffer
  deactivated.}
\vspace{2ex}
\begin{tabular}{l|c|cccc}
  \hline
  \hline
  Test & Run & System & User & Elapsed &  Elapsed \\
       & \#  &   [s]  &  [s] &   [s]   &   [\%]   \\
  \hline
        System {\tt cat}  &   1  &   3.6  &   0.1  &  31.9  &  53.6 \\
                          &   2  &   0.6  &   0.0  &   0.6  &   0.9 \\
  \hline
   {\tt classic} program  &   3  &   2.2  &   4.0  &  35.5  &  59.7 \\
                          &   4  &   0.6  &   1.9  &   2.5  &   4.3 \\
  \hline
       No virtual buffer  &   5  &   4.3  &  53.4  &  59.5  & \bf{100.0} \\
                          &   6  &   1.2  &  52.8  &  54.0  & \bf{90.7} \\
  \hline
    Small virtual buffer  &   7  &   4.4  &  55.9  &  62.3  & 104.6 \\
                          &   8  &   1.3  &  55.7  &  57.1  &  95.8 \\
                          &   9  &   1.2  &  55.7  &  56.8  &  95.4 \\
                          &  10  &   1.1  &  55.6  &  56.6  &  95.1 \\
  \hline
    Large virtual buffer  &  11  &   7.9  &  55.1  &  67.9  & \bf{114.0} \\
                          &  12  &   0.0  &  45.5  &  45.5  & \bf{76.4} \\
                          &  13  &   2.9  &  55.8  &  59.7  & 100.2 \\
                          &  14  &   0.0  &  45.5  &  45.5  &  76.5 \\
  \hline
\end{tabular}
\label{tab:wilma-classicv2}
\end{table}

\begin{table}[p]
\centering
\caption{System, user, and elapsed times for reading the FTS data file
  with the various tests described in Table~\ref{tab:descr}. In the
  column Elapsed [\%], the reference (100\%) is taken when reading the
  file the first time with a \class{} procedure with the virtual
  buffer deactivated.}
\vspace{2ex}
\begin{tabular}{l|c|cccc}
  \hline
  \hline
  Test & Run & System & User & Elapsed &  Elapsed \\
       & \#  &   [s]  &  [s] &   [s]   &   [\%]   \\
  \hline
        System {\tt cat}  &   1  &   4.0  &   0.1  &  35.2  & 100.3 \\
                          &   2  &   0.6  &   0.0  &   0.6  &   1.8 \\
  \hline
   {\tt classic} program  &   3  &   3.3  &   1.5  &  34.7  &  99.1 \\
                          &   4  &   0.7  &   0.4  &   1.1  &   3.2 \\
  \hline
       No virtual buffer  &   5  &   1.6  &  18.9  &  35.0  & \bf{100.0} \\
                          &   6  &   0.8  &  16.8  &  17.6  & \bf{50.3} \\
  \hline
    Small virtual buffer  &   7  &   1.7  &  20.1  &  34.6  &  98.6 \\
                          &   8  &   0.9  &  18.0  &  18.9  &  54.0 \\
                          &   9  &   0.8  &  18.0  &  18.8  &  53.7 \\
                          &  10  &   0.7  &  18.0  &  18.7  &  53.5 \\
  \hline
    Large virtual buffer  &  11  &  14.3  &  16.2  &  46.7  & \bf{133.4} \\
                          &  12  &   0.2  &  17.2  &  17.3  & \bf{49.5} \\
                          &  13  &   1.8  &  18.7  &  23.2  &  66.2 \\
                          &  14  &   0.1  &  17.2  &  17.3  &  49.5 \\
  \hline
\end{tabular}
\label{tab:fts-classicv2}
\end{table}

\subsection{Best \classic{} performances}
\label{txt:classic}

We run here the simplest possible program based on the \classic{}
library. This means that the ``technical'' bytes of the \classic{}
Data Container are interpreted, but not the \class{} data
themselves. This program aims to measure what its the fastest speed
the application can reach if it would have no overheads. The results
are shown in the Tables~\ref{tab:wilma-classicv2} and
\ref{tab:fts-classicv2}, runs 3 and 4.\\

Comparing the runs 1 and 3 (\ie\ file not in system cache), we see
that the \classic{} library is as fast as\footnote{Actually, the
  \classic{} program is slightly faster. We know that accesses are
  optimum with 4 kB records (equal to sector size). The {\tt cat}
  command may not perform as efficient accesses.} the command {\tt
  cat} when the data is larger than the header (FTS data). ``as fast
as'' means that the bottleneck is the reading from the hard drive,
which limits both applications. On the other hand, when the header and
data have comparable sizes, we can see overheads (11\% for WILMA file)
probably due to the much more complex access to the headers than to
the data. Remember that the WILMA file offers more than 1.5 million of
headers while FTS data has about 50 thousands of headers, for an
equivalent file size on disk.\\

Comparing runs 2 and 4 (\ie\ file in system cache), we see a longer
access time in all cases, probably due to the complex header accesses
detected above.

\subsection{\class{} without its virtual buffer}
\label{txt:class-novirt}

We run here a dedicated \class{} procedure which opens the file and
performs a {\tt GET} loop. Times are measured for this loop only, \ie\
the times for {\tt FILE IN} and {\tt FIND} are excluded (this means
they do not strictly compare to the {\tt cat} and \classic{} reading
above). For this test, the \class{} virtual buffer ({\tt SET VIRTUAL
  ON|OFF}) is deactivated. The results are shown in the
Tables~\ref{tab:wilma-classicv2} and \ref{tab:fts-classicv2}, runs 5
and 6. The run 5 is considered to be the reference (100\%) in all
these benchmarks: the file is not yet in the system cache, and the
\class{} virtual buffer is deactivated for reading and writing.\\

Comparing the runs 3 and 5 (\ie\ file not in system cache), we can see
again that the reading times are almost equal when the data is large
compared to the headers (FTS). The {\tt GET} command overheads are
neglictible in this case. On the other hand, they almost double the
reading time for WILMA data.\\

Comparing the runs 5 and 6 (\ie\ file not in system cache, then in
system cache) we can see that the benefit of the system cache is large
for FTS data (about 50\%). This benefit is smaller (about 10\%) in
case of WILMA data because of the {\tt GET} command overheads.

\subsection{\class{} with small virtual buffer}

We run here the same procedure as before, but we activate here the
virtual buffer. However, we choose a limited size (1024 MBytes) which
is smaller than the file size. Since the \class{} virtual is a
\emph{fifo}, this basically implies that the buffer is filled and
overwritten several times, but nothing is re-read from it at next run
(since we loop in the same order). The results are shown in the
Tables~\ref{tab:wilma-classicv2} and \ref{tab:fts-classicv2}, runs 7,
8, 9 and 10.\\

Comparing runs 5 and 7, we see that saving the observations in the
\class{} virtual buffer adds a penalty. This is expected because of
the extra-work performed in the {\tt GET} command.\\

Comparing runs 6 and 8, we see that reading from the virtual buffer is
always slower than reading from the system cache only, because the
buffer is too small, we do not actually re-read anything from it. In
other words, we are always exposed to the buffering penalty described
above.\\

Restarting with an empty virtual buffer (run 9 and 10) does not change
the situation as in practice the runs 8, 9 and 10 are always
performing the same operation: they have comparable elapsed times.

\subsection{\class{} with large virtual buffer}

Again we run here the procedures with the \class{} virtual buffer, but
it is now large enough so that all the observations can be
buffered. This \emph{should} be THE favorable case where the buffer
benefits are visible.\\

However, the concept of \emph{large enough} buffer is tricky. There
are several things to know:
\begin{enumerate}
\item \class{} splits the virtual buffer (size {\tt SPACE\_CLASS})
  into 2 sub-buffers: one for the headers, one for the data. These
  sub-buffers are isolated, \ie\ you can have one header buffered but
  not its data, or vice-versa, or both.
\item \class{} allocates arbitrarily 25\% of {\tt SPACE\_CLASS} for
  the header buffering, and 75\% for the data buffering. This can be
  discussed, in particular in front of nowadays ratio between header
  and data.
\item \class{} buffers (\ie\ copies) the \emph{memory} headers
  (Fortran type {\tt header}) which (current) size is 1318 words.
\end{enumerate}
The last point above leads to 7.8 GB needed to buffer all the headers
of the WILMA test data (to be compared to the 4.2 GB of the file),
so {\tt SPACE\_CLASS} has to be set to 31.2 GB at least.\\

The results are shown in the Tables~\ref{tab:wilma-classicv2} and
\ref{tab:fts-classicv2}, runs 11, 12, 13 and 14. Comparing the runs 5
and 11, we see again the penalty to save the observations in the
virtual buffer. It can be from 10\% up to 33\% for FTS data. Comparing
the runs 6 and 12, we now see the benefits of the virtual buffer
compared to the system cache: about 15\% for WILMA data, about 1\% for
FTS. When restarting the runs with an empty virtual buffer (runs 13
and 14), we see again the first buffering penalty.

\section{Conclusion}

The results exposed in the previous sections shows that the virtual
buffer can offer some benefits. However, with nowadays observations
(data much larger than header), the benefit seems reduced. Plus, there
are a lot conditions to meet before actually experiencing an
improvement. Pros and cons are detailed below.\\

%
Pros:
\begin{itemize}
\item when all the conditions are met, yes, the \class{} virtual
  buffer offers a benefit (larger then the system cache) when
  accessing a second time the observations.
\item the \class{} virtual buffer is always available, on all
  platforms, while the system cache may not be available (\eg\ system
  out of memory, kernel's choice to cache one file and not the
  other\footnote{See section \ref{txt:laptop}.}).
\end{itemize}
Note that we compare here the efficiency of accessing the system cache
vs the \class{} virtual buffer. The disk efficiency (rotational, SSD)
is not involved once the observation has been read.\\

%
Cons:
\begin{itemize}
\item the first buffering penalty is always present, and it can be
  quite large. On the other hand, the benefits are only available the
  second time the data is read, if needed. Depending on its reduction
  procedures, the user may never see the benefits of the virtual
  buffer.
\item the virtual buffer is reset from one \class\ session to another,
  it is independent for all parallel jobs. Also, each {\tt FILE
    IN|BOTH|UPDATE} command resets the virtual buffer (because
  identifiers are the entry number in the current file, there is no
  memory of which file it comes from). Again, depending on its
  reduction procedures, the user may never see the benefits of the
  virtual buffer.
\item the default {\tt SPACE\_CLASS} (128 MiB) can probably be
  considered as a \emph{small buffer} (i.e. no benefits, only
  penalties) precisely for the files where buffering is the most
  needed. Enlarging the default is possible but the memory cost may
  not be acceptable everywhere. Actually the best should be a
  per-machine or ideally per-file {\tt SPACE\_CLASS}. In other words,
  the current implementation is not dynamic enough.
\item the virtual buffer allocates a non-negligible amount of memory
  ({\tt SPACE\_CLASS}) which could be used for other purposes
  (e.g. for other programs, or for caching more files at the system
  level).
\item memory cost for caching headers in the virtual buffer is huge,
  much more than their actual size on disk. This means that the header
  in the system cache has less cost than in the virtual buffer. This
  could be optimized but with a (complicated?) management of present
  sections.
\item the ratio 25\%/75\% for splitting {\tt SPACE\_CLASS} for headers
  and data may not be optimum, and may lead to extreme {\tt
    SPACE\_CLASS} values compared to the file size! We can think in
  making this ratio dynamic each time a file is opened (\ie\ each time
  we know how many observations are on disk).
\item as of today the virtual buffer requires the {\tt type(header)}
  to be static (predictible size, contiguous in memory). It can not
  hold dynamic sections (user section, associated arrays, arbitrary
  number of windows in the base section) as long as ``static'' headers
  are used. This is a serious limitation for Class developers.
\item ``static'' headers can be replaced by ``dynamic'' headers, but
  this means redesigning the buffering of the headers (knowing each
  individual size) so that they always fit in 25\% of {\tt
    SPACE\_CLASS}.
\end{itemize}

Because of all the conditions which are needed to benefit a useful
virtual buffer, and because of the too many cons, in particular for
further improvements of the \class\ Data Format, the current
implementation of the virtual buffer was removed from \class\ on the
July $26^{th}$, 2017. If needed, it might be reimplemented in the
future using a more flexible/modern approach.

\newpage
\section{Complementary tests}

\subsection{Impact of the \classic{} binary container}

The same tests are run with the same WILMA data, using a \classic{}
binary container version 1 (\ie\ 128-words records). File size is
4,735 MBytes in this case for the same data. The main conclusions are
identical: the virtual buffering has a non negligible cost, and the
benefit is about 20\% at second read time.

\begin{table}[htb]
\centering
\caption{Same as Table~\ref{tab:wilma-classicv2}, but the WILMA data
  file uses \classic{} binary container version 1.}
\vspace{2ex}
\begin{tabular}{l|c|cccc}
  \hline
  \hline
  Test & Run & System & User & Elapsed &  Elapsed \\
       & \#  &   [s]  &  [s] &   [s]   &   [\%]   \\
  \hline
        System {\tt cat}  &   1  &   4.1  &   0.1  &  35.9  &  57.7 \\
                          &   2  &   0.6  &   0.0  &   0.6  &   1.0 \\
  \hline
   {\tt classic} program  &   3  &   3.2  &   5.3  &  41.9  &  67.4 \\
                          &   4  &   0.9  &   2.6  &   3.5  &   5.7 \\
  \hline
       No virtual buffer  &   5  &   5.3  &  55.8  &  62.2  & \bf{100.0} \\
                          &   6  &   1.8  &  55.1  &  56.9  & \bf{91.4} \\
  \hline
    Small virtual buffer  &   7  &   5.7  &  57.5  &  64.4  & 103.5 \\
                          &   8  &   1.9  &  56.9  &  58.9  &  94.7 \\
                          &   9  &   1.9  &  56.9  &  58.8  &  94.4 \\
                          &  10  &   1.7  &  57.2  &  58.9  &  94.7 \\
  \hline
    Large virtual buffer  &  11  &   9.2  &  56.7  &  68.8  & \bf{110.5} \\
                          &  12  &   0.0  &  45.3  &  45.3  & \bf{72.8} \\
                          &  13  &   5.6  &  57.5  &  63.6  & 102.2 \\
                          &  14  &   0.0  &  45.3  &  45.3  &  72.8 \\
  \hline
\end{tabular}
\label{tab:wilma-classicv1}
\end{table}

\subsection{Impact of the compiler}

The Table~\ref{tab:ifort} shows that the Intel Fortran compiler has an
impact on the results, probably because of the different buffering
strategies between gfortran and ifort. We see that the \class{}
virtual buffer has 8\% penalty at first read, and the benefit is 4\%
compared to the system cache at the second read (FTS data).

\begin{table}[htb]
\centering
\caption{Same as Table~\ref{tab:fts-classicv2} (subset), but \class{}
  compiled with ifort 11.1}
\vspace{2ex}
\begin{tabular}{l|c|cccc}
  \hline
  \hline
  Test & Run & System & User & Elapsed &  Elapsed \\
       & \#  &   [s]  &  [s] &   [s]   &   [\%]   \\
  \hline
       No virtual buffer  &   5  &   3.9  &  26.7  &  34.5  & \bf{100.0} \\
                          &   6  &   1.2  &  25.8  &  27.0  & \bf{78.2} \\
  \hline
    Large virtual buffer  &  11  &   7.1  &  26.2  &  37.3  & \bf{108.2} \\
                          &  12  &   0.0  &  25.7  &  25.7  & \bf{74.7} \\
                          &  13  &   2.4  &  26.6  &  30.5  &  88.5 \\
                          &  14  &   0.0  &  25.7  &  25.7  &  74.6 \\
  \hline
\end{tabular}
\label{tab:ifort}
\end{table}

\newpage
\subsection{Portable hardware}
\label{txt:laptop}

A part of the tests are performed on a laptop, assumed to represent an
hardware with limited capabilities: a Dell Latitude E6430s with 8 GB
RAM memory, using a Fedora 25 with gfortran 6.3.1. The results are
exposed in the Table~\ref{tab:laptop}. This test is interesting in the
sense that the system {\tt cat} command shows no improvement during
the second run. Actually when looking closely at the benchmark
measurements (not provided here), the system cache benefit is seen for
the first file processed, but not for the subsequent ones. It seems
that by default on this OS, the system cache is configured as some
kind of \emph{filo}\footnote{The Linux kernel offers a lot of options
  to fine tune the system cache, overriding the default
  behaviour.}. There is a benefit but it is not seen here because of
the strategy used for this benchmark (measurement repeated done for 10
different files) and the median computed from the results.

\begin{table}[htb]
\centering
\caption{Same as Table~\ref{tab:fts-classicv2} (subset), but running
  on a laptop.}
\vspace{2ex}
\begin{tabular}{l|c|cccc}
  \hline
  \hline
  Test & Run & System & User & Elapsed &  Elapsed \\
       &  \#  &   [s]  &  [s] &   [s]   &   [\%]   \\
  \hline
        System {\tt cat}  &   1  &   2.0  &   0.0  &  42.3  & 100.0 \\
                          &   2  &   2.1  &   0.0  &  42.2  &  99.8 \\
  \hline
       No virtual buffer  &   5  &   2.2  &  18.7  &  42.3  & \bf{100.0} \\
                          &   6  &   2.3  &  19.2  &  41.8  & \bf{98.8} \\
  \hline
    Small virtual buffer  &   7  &   2.5  &  22.1  &  42.8  & 101.2 \\
                          &   8  &   2.6  &  22.1  &  42.1  &  99.6 \\
                          &   9  &   1.9  &  18.6  &  28.9  &  68.2 \\
                          &  10  &   0.8  &  15.6  &  16.4  &  38.8 \\
  \hline
    Large virtual buffer  &  11  &   2.5  &  21.4  &  42.6  & \bf{100.7} \\
                          &  12  &   0.0  &  13.7  &  13.8  & \bf{32.5} \\
                          &  13  &   3.8  &  20.6  &  42.2  &  99.7 \\
                          &  14  &   0.0  &  13.7  &  13.8  &  32.6 \\
  \hline
\end{tabular}
\label{tab:laptop}
\end{table}

\newpage
\subsection{Mac OSX}

The tests are also performed on another operating system, in order to
check if the system cache shows similar performances. The system runs
Mac OSX Sierra (10.12.5) with 16 GB of RAM memory. The working
partition is a logical partition mixing rotational hard disk and SSD
disk. \class{} is compiled with gfortran 6.3.0 (MacPorts). The
conclusions are similar to the Scientific Linux operating system.

\begin{table}[htb]
\centering
\caption{Same as Table~\ref{tab:fts-classicv2} (subset), but running
  under Mac OSX.}
\vspace{2ex}
\begin{tabular}{l|c|cccc}
  \hline
  \hline
  Test & Run & System & User & Elapsed &  Elapsed \\
       & \#  &   [s]  &  [s] &   [s]   &   [\%]   \\
  \hline
        System {\tt cat}  &   1  &   1.7  &   0.0  &  54.1  & 100.0 \\
                          &   2  &   0.7  &   0.0  &   0.7  &   1.4 \\
  \hline
       No virtual buffer  &   5  &   2.8  &  14.3  &  54.0  & 100.0 \\
                          &   6  &   1.0  &  12.3  &  13.9  &  25.7 \\
  \hline
    Small virtual buffer  &   7  &   2.8  &  14.8  &  58.9  & 109.0 \\
                          &   8  &   1.1  &  13.0  &  15.4  &  28.6 \\
                          &   9  &   1.4  &  13.1  &  14.5  &  26.9 \\
                          &  10  &   1.0  &  12.8  &  13.8  &  25.6 \\
  \hline
    Large virtual buffer  &  11  &   3.5  &  15.1  &  55.6  & 102.9 \\
                          &  12  &   0.1  &  11.8  &  11.8  &  21.9 \\
                          &  13  &   2.3  &  13.5  &  16.8  &  31.2 \\
                          &  14  &   0.1  &  11.8  &  11.8  &  21.9 \\
  \hline
\end{tabular}
\label{tab:macosx}
\end{table}

\subsection{Impact of SSD disk}

The tests are also performed on the reference machine, but using a SSD
drive\footnote{SAMSUNG 2TB}. The results are shown in the
Table~\ref{tab:ssd}. We can see that the absolute times when reading
from disk have been decreased (typically from 35 seconds to
25)\footnote{The benefit is not that huge, probably due to the SATA
  controler limitations.}. On the other hand, the accesses from system
cache or virtual buffer are not modified; this is expected. As a
consequence, the relative gains between from-disk and from-memory
reads are worse.

\begin{table}[htb]
\centering
\caption{Same as Table~\ref{tab:fts-classicv2} (subset), but reading files
  from a SSD disk.}
\vspace{2ex}
\begin{tabular}{l|c|cccc}
  \hline
  \hline
  Test & Run & System & User & Elapsed &  Elapsed \\
       & \#  &   [s]  &  [s] &   [s]   &   [\%]   \\
  \hline
        System {\tt cat}  &   1  &   4.1  &   0.1  &  25.1  &  96.1 \\
                          &   2  &   0.6  &   0.0  &   0.6  &   2.3 \\
  \hline
   {\tt classic} program  &   3  &   3.2  &   1.6  &  25.1  &  96.2 \\
                          &   4  &   0.6  &   0.4  &   1.0  &   3.9 \\
  \hline
       No virtual buffer  &   5  &   3.1  &  18.6  &  26.1  & \bf{100.0} \\
                          &   6  &   0.8  &  17.0  &  17.8  & \bf{68.2} \\
  \hline
    Small virtual buffer  &   7  &   3.1  &  19.5  &  27.0  & 103.2 \\
                          &   8  &   0.8  &  18.0  &  18.9  &  72.3 \\
  \hline
    Large virtual buffer  &  11  &   3.1  &  19.8  &  27.2  & \bf{104.2} \\
                          &  12  &   0.0  &  17.4  &  17.4  & \bf{66.5} \\
  \hline
\end{tabular}
\label{tab:ssd}
\end{table}


% \subsection{Reading from high speed network}
% 
% We also check reading from a 1 Gb/sec network.  ZZZ Pas de conclusion
% particuliere, le fichier sort d'un device comme un aute et est mis en
% cache memoire localement.
% 
% \begin{table}[htb]
% \centering
% \caption{Same as Table~\ref{tab:fts-classicv2} (subset), but reading files
%   from a 1 Gb/sec network.}
% \vspace{2ex}
% \begin{tabular}{l|c|cccc}
%   \hline
%   \hline
%   Test & Run & System & User & Elapsed &  Elapsed \\
%        & \#  &   [s]  &  [s] &   [s]   &   [\%]   \\
%   \hline
%         System {\tt cat}  &   1  &   2.3  &   0.0  &  42.6  &  50.6 \\
%                           &   2  &   0.6  &   0.0  &   0.6  &   0.7 \\
%   \hline
%    {\tt classic} program  &   3  &   2.3  &   1.0  &  42.3  &  50.3 \\
%                           &   4  &   0.6  &   0.4  &   1.0  &   1.2 \\
%   \hline
%        No virtual buffer  &   5  &   1.8  &  19.1  &  84.1  & 100.0 \\
%                           &   6  &   0.8  &  17.1  &  17.8  &  21.2 \\
%   \hline
%     Large virtual buffer  &  11  &   2.4  &  20.1  &  81.7  &  97.1 \\
%                           &  12  &   0.0  &  17.5  &  17.5  &  20.8 \\
%                           &  13  &   1.6  &  18.4  &  28.3  &  33.6 \\
%                           &  14  &   0.0  &  17.4  &  17.4  &  20.7 \\
%   \hline
% \end{tabular}
% \label{tab:network}
% \end{table}

\end{document}

%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
