\documentclass{article}

\usepackage{graphicx}
\usepackage{fancyhdr}
\usepackage{amssymb}   % For natural numbers N
\usepackage{afterpage}
\usepackage{enumitem}  % Allows to resume list numbering

\makeindex{}

\makeatletter
\textheight 625pt
\textwidth 460pt
\oddsidemargin 0pt
\evensidemargin 0pt
\marginparwidth 50pt
\topmargin 0pt
\brokenpenalty=10000

\newcommand{\superscript}[1]{\ensuremath{^{\textrm{#1}}}}
\newcommand{\subscript}[1]{\ensuremath{_{\textrm{#1}}}}

\newcommand{\ie} {{\it i.e.}}
\newcommand{\eg} {{\it e.g.}}
\newcommand{\cf} {cf.}

\newcommand{\gildas}  {\mbox {\bf GILDAS}}
\newcommand{\class}   {\mbox {\bf CLASS}}
\newcommand{\clic}    {\mbox {\bf CLIC}}
\newcommand{\mrtcal}  {\mbox {\bf MRTCAL}}
\newcommand{\classic} {\mbox {\bf CLASSIC}}
\newcommand{\cdc}     {\mbox {\bf CLASSIC} Data Container}
\newcommand{\clib}    {\mbox {\bf CLASSIC} Library}
\newcommand{\classdf} {\mbox {\bf CLASS} Data Format}
\newcommand{\clicdf}  {\mbox {\bf CLIC} Data Format}

% Makes fancy headings and footers.
\pagestyle{fancy}

\fancyhf{} % Clears all fields.

\rhead{\scshape \nouppercase{\rightmark}}   %
\fancyfoot[C]{\bfseries \thepage{}}

\renewcommand{\headrulewidth}{0pt}    %
\renewcommand{\footrulewidth}{0pt}    %

\renewcommand{\sectionmark}[1]{%
  \markright{\thesection. \MakeLowercase{#1}}}
\renewcommand{\subsectionmark}[1]{}

%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%

\newcommand{\TabFileDesc}{%
    \begin{table}[tb]
      \begin{center}
        \caption{File Descriptor Version 2}
        \vspace{1ex}
        \begin{tabular}{l|cl|l|l}
          Position        & Parameter      & Fortran Kind & Purpose                               & Unit    \\
          \hline
          1               & {\tt code}     & Character*4  & File code                             &  -      \\
          2               & {\tt reclen}   & Integer*4    & Record length                         & words   \\
          3               & {\tt kind}     & Integer*4    & File kind                             &  -      \\
          4               & {\tt vind}     & Integer*4    & Index version                         &  -      \\
          5               & {\tt lind}     & Integer*4    & Index length                          & words   \\
          6               & {\tt flags}    & Integer*4    & Bit flags. \#1: single or multiple,   &  -      \\
                          &                &              & \#2-32: provision (0-filled)          &         \\
          \hline
          7:8             & {\tt xnext}    & Integer*8    & Next available entry number           &  -      \\
          9:10            & {\tt nextrec}  & Integer*8    & Next record which contains free space & record  \\
          11              & {\tt nextword} & Integer*4    & Next free word in this record         & word    \\
          \hline
          12              & {\tt lex1}     & Integer*4    & Length of first extension index       & entries \\
          13              & {\tt nex}      & Integer*4    & Number of extensions                  &  -      \\
          14              & {\tt gex}      & Integer*4    & Extension growth rule                 &  -      \\
          15:{\tt reclen} & {\tt aex(:)}   & Integer*8    & Array of extension addresses          & record
        \end{tabular}
        \label{tbl:fildesc}
      \end{center}
    \end{table}}

\newcommand{\TabClassicCodes}{%
  \begin{table}[htb]
    \begin{center}
      \caption{\cdc{} encoding of the system and kind. Most of the recent
        computers are little endian (IEEE) machines. The (very) old PDP-11
        files (with Integer*2 words) are recognized but not supported
        anymore (XX means 2 non-blank characters). Note that in V2 files,
        the single/multiple attribute is not encoded anymore in this code.}
      \vspace{1ex}
      \begin{tabular}{c|cccc}
        Kind        & IEEE               & EEEI               & VAX                     & PDP-11 \\
        \hline
        V1 multiple & `1A$\sqcup\sqcup$' & `1B$\sqcup\sqcup$' & `1$\sqcup\sqcup\sqcup$' & `1$\sqcup$XX' \\
        V1 single   & `9A$\sqcup\sqcup$' & `9B$\sqcup\sqcup$' & `9$\sqcup\sqcup\sqcup$' &  N/A          \\
        V2          & `2A$\sqcup\sqcup$' & `2B$\sqcup\sqcup$' & `2$\sqcup\sqcup\sqcup$' &  N/A
      \end{tabular}
      \label{tbl:classiccodes}
    \end{center}
  \end{table}}

\newcommand{\TabAppliKinds}{%
  \begin{table}[htb]
    \begin{center}
      \caption{\cdc{} encoding of the owner ({\tt kind} parameter in the
        File Descriptor). These values are NOT let free to the calling
        application to avoid conflicts.}  \vspace{1ex}
      \begin{tabular}{cc}
        Owner & Identifier \\
        \hline
        \class  & 1 \\
        \clic   & 2 \\
        \mrtcal & 3
      \end{tabular}
      \label{tbl:kinds}
    \end{center}
  \end{table}}

\newcommand{\TabEntryDesc}{%
    \begin{table}[hb]
      \begin{center}
        \caption{Entry Descriptor Version 2}
        \vspace{1ex}
        \begin{tabular}{lcl|l|l}
          Position      & Parameter             & Fortran Kind & Purpose                                       & Unit  \\
          \hline
          1             & {\tt code}            & Character*4  & Identifier of Entry Descriptors               &  -    \\
          2             & {\tt version}         & Integer*4    & Observation version                           &  -    \\
          3             & {\tt nsec}            & Integer*4    & Number of sections                            &  -    \\
          4:5           & {\tt nword}           & Integer*8    & Length of this entry (descriptor included)    & words \\
          6:7           & {\tt adata}           & Integer*8    & Data address                                  & word  \\
          8:9           & {\tt ldata}           & Integer*8    & Data length                                   & words \\
          10:11         & {\tt xnum}            & Integer*8    & Entry number                                  &  -    \\
          12:...        & {\tt seciden(1:nsec)} & Integer*4    & Section numbers (identifiers)                 & code  \\
          ...:...       & {\tt secleng(1:nsec)} & Integer*8    & Section lengths                               & words \\
          ...:11+5*nsec & {\tt secaddr(1:nsec)} & Integer*8    & Section addresses                             & word  \\
          ...:11+5*msec & (unused)              &              & Empty room for up to 'msec' sections          &
        \end{tabular}
        \label{tbl:entdesc}
      \end{center}
    \end{table}}

%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%

\begin{document}

\title{IRAM Memo 2013-2\\[3\bigskipamount]
  \classic{} Data Container}
\author{S. Bardeau$^1$, V. Pi\'etu$^1$, J. Pety$^{1,2}$\\[0.5cm]
  1. IRAM (Grenoble) \\
  2. LERMA, Observatoire de Paris}
\date{October, 3$^{rd}$ 2013\\
  Version 1.0}

\maketitle{} %

\begin{abstract}
  The \class{}/\clicdf{} are digital formats used to describe
  single-dish/interferometric radio-astronomy data. They can be described
  in two layers: 1) a \cdc{}, which is generic enough to store many kind of
  data, typically several {\it observations} which gather observational
  parameters with actual data, and 2) the \class{}/\clicdf{} itself, which
  make a particular use of the \cdc{}.  
  
  The size of the datasets produced by the IRAM instruments experience a
  tremendeous increase (because of multi-beam receivers, wide bandwidth
  receivers, spectrometers with thousands of channels, and/or new observing
  mode like the interferometric on-the-fly).  This implied that the
  \class{}/\clicdf{} were reaching common limits in the size of data which
  could be stored. To solve these issues, the \cdc{} standard was revised.
  This documents aims to describe the new standard. A companion document
  describes the \gildas{} library which implements this standard and which
  is now used by \class{} and \clic{}.\\

Related documents: The \clib{}, IRAM memo 2013-3
\end{abstract}

\newpage{}
\tableofcontents{}

%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
\newpage{}

We describe here the \cdc{} standard Version 2. Its historical version
(Version 1) and the difference between the two versions are detailed in the
section \ref{txt:cdcv1}.  Only the \cdc{} standard is detailed here; the
\gildas{} implementation of this standard (e.g. reading and writing) is
described in an other memo about the {\it CLASSIC Library}. We will name
\emph{application} the \gildas{} programs (e.g., \class{}, \clic{}, ...)
that use the \cdc{} standard.

\section{Overview}

\begin{figure}[p]
  \centering \includegraphics[width=\textwidth]{classic-data-container}
  \caption{\cdc{} diagram. In this example, the file has N
    extensions. Extensions use an exponential growth mode (1, 2, ... 8
    records devoted to the extension indexes). The last extension is
    incomplete and contains only 5 file entries, while there is room for 32
    in the extension index. The file entries have different number of
    sections, and sections and data have variable lengths.}
  \label{fig:diagram}
\end{figure}

Figure~\ref{fig:diagram} tries to reproduce at best the structure of a
\cdc{} file. This can be summarized as follows:\\
1 File = 1 File Descriptor + 0 or more Extensions,
\begin{itemize}
\item File descriptor: contains file information (system type, record
  length, addresses of extension among others).
\item Extension = 1 Extension Index + 1 or more File Entries,
  \begin{itemize}
  \item Extension Index: for each entry of the extension, it contains
    a few criteria characterizing an entry (to allow quick entry
    search) and the address of the entry
  \item Entry = 1 Entry Descriptor + 1 Observation,
    \begin{itemize}
    \item Entry Descriptor: contains parameters describing the entry (e.g.
      number of sections and their addresses, address and length of data).
    \item Observation = 0 or 1 Observation Header + 0 or 1 Data array
      \begin{itemize}
      \item Observation Header = 0 or more Sections
      \item Data array: the data themselves
      \end{itemize}
    \end{itemize}
  \end{itemize}
\end{itemize}
The extension index serves to index some important information about the
file entries. For convenience, we also define the concept of {\it File
  Index}, which is the (virtual) collection of all the indexes of all the
extensions in the file. The \cdc{} enforces the following rules:
\begin{enumerate}
\item the File Descriptor is always found in the first record in the file.
\item the various elements involved in the data container are found thanks
  to the addresses provided by their parent elements. Any other mean or
  assumption is not guaranteed to work.
\item the File Entries have an entry number which is their rank in the File Index.
\end{enumerate}

\section{Detailed description}

A \classic{} file is a Fortran, Direct-Access, Unformatted (binary) file
satisfying the so-called \cdc{}. Such files are usually read and written by
a Fortran library and accessed {\it record} by {\it record} of equal
length. A \classic{} file contains only (in the following order):
\begin{enumerate}
\item a {\it File Descriptor},
\item zero or more {\it Extensions}.
\end{enumerate}

\subsection{The File Descriptor}
\label{txt:fildesc}

\TabFileDesc{} %
\TabClassicCodes{} %
\TabAppliKinds{} %

Table~\ref{tbl:fildesc} summarises the contents of the File Descriptor. It
always spreads (entirely and only) over the first record of the file. It
describes how the file contents are stored. It is composed of 3 groups of
parameters:
\begin{itemize}
\item The first group describes the parameters tunable by the application,
  and which will affect the whole file.
  \begin{enumerate}
  \item the first 4 bytes of the file are a {\tt code} identifying the
    files.  They also provide the bit encoding system, as detailed in
    Table~\ref{tbl:classiccodes}.
  \item the next word ({\tt reclen}) is the record length of this
    Fortran direct-access file.  It is stored in 4-bytes word
    units\footnote{A {\it 4-bytes word} is defined here as a block of
      4 consecutive bytes.  It is the basic unit for all the kind of
      variables involved here, \eg\ a Fortran Integer*4 uses 1 word
      and a Fortran Character*12 uses 3 words.}. There is no
    restrictions for {\tt reclen} other than the Fortran ones, except
    that at least one extension address should fit, \ie\ {\tt reclen}
    must be at least 16 words. It can be a non-power of 2, it can be
    odd, but programmer should keep in mind file access efficiency.
  \item the {\tt kind} of the file is stored next (e.g. \class{}, \clic{},
    or any other application). This helps the reading application to detect
    whether it opens a file which it is able to read. In order to avoid
    conflicts, the possible values are NOT let free to the applications
    (see Table~\ref{tbl:kinds}).
  \item the next 2 words are the Index Version and Length (see section
    \ref{txt:extensionindex}). The Index is a program-specific object, and
    it is likely to evolve through time. Note that there is no restriction
    on the Index Length (it can be a non-power of 2, or odd).
  \item the next word is used as 32 bit-flags, describing some boolean
    attributes of the file. Currently only the first\footnote{{\it first}
      in the sense of the Fortran bit model, \ie\ processor-independent.}
    bit is used to indicate if the observation numbering is single or
    multiple. The other 31 bits are provisioned and set to 0, indicating
    the historical status of some future attributes.
  \end{enumerate}
\item The 2nd group indicates where the next free memory positions are
  available.
  \begin{enumerate}[resume]
  \item {\tt xnext} provides the next free entry number. File entries
    are numbered continuously from 1 to {\tt xnext-1}.
  \item {\tt nextrec} identifies the record at the end of the file where
    space is still available to append new data.
  \item {\tt nextword} indicates where this space starts in the {\tt
      nextrec} record.
  \end{enumerate}
\item The last group describes how the size of extensions increases and the
  extension addresses.
  \begin{enumerate}[resume]
  \item {\tt lex1} is the length of the first extension, in number of
    entries. This length can vary from one file to another, and can be
    considered as a scaling factor for the extensions.
  \item {\tt nex} is the number of extensions currently in use in the file.
    It can increase or not when an entry is appended to the file.
  \item {\tt gex} describes the extension growth (see below).
  \item finally {\tt aex(:)} is an array providing the starting record
    number for each extension index. Since they are stored as Fortran
    Integer*8 values, 2 words are needed for each address. Given the fact
    that the {\tt aex(:)} array fills the 1st record up to its end, it is
    limited to $nex = (reclen-14)/2$ elements, which is the maximum number
    of extensions ({\tt mex}) in the file.
  \end{enumerate}
\end{itemize}
The extension can either all have the same size or exponentially increase
their size. Both cases can be summarized as
\begin{equation}
  \label{equ:lexe}
  lex_E(iex) = lex1 \times m^{iex-1}
\end{equation}
where $lex_E(iex)$ is the size of the {\tt iex}-th extension index in
number of file entries. The total number of file entries which can be
stored in a file with {\tt nex} extensions is then:
\begin{equation}
  \label{equ:nentry}
  N_{entry} = lex1 \times (m^0 + m^1 + m^2 + ... + m^{nex-1})
\end{equation}
If the mantissa $m$ is 1, all the extensions have the same size ($N_{entry}
= lex1 \times nex$). If $m>1$, the growth is exponential ($N_{entry} = lex1
\times (2^{nex}-1)$ if $m=2$). Only equal size extensions were used in the
version 1 of the \cdc. The new exponential growth solves two problems: 1)
the file size will remain small when it contains only a few file entries
(no pre-allocation of a large index), and 2) there is virtually no more
limit on the number of entries the file can hold\footnote{In practice, the
  number of file entries will be limited to $2^{63}-1$, the largest integer
  number coded as a IEEE Integer*8, \eg\ $mex = 63$ if $m = 2$}.

If $m$ is non-integer, floating point computations should be involved. In
order to leave no room for round-off problems or ambiguity, the following
rules shall be enforced:
\begin{enumerate}
\item $m \ge 1$.
\item $m$ is limited to 1 significant digit after the coma.
\item the number of records devoted to each extension index must be rounded
  to the lowest upper-or-equal integer value large enough to store the
  $lex_E(iex)$ entry indexes:
  \begin{equation}
    \label{equ:lexr}
    lex_R(iex) = \mathrm{ceiling} \left( \frac{lex_E(iex) \times lind}{reclen} \right)
  \end{equation}
\end{enumerate}
Given these rules, the extensions growth can easily be implemented with
integer multiplications and divisions. To simplify the application of rule
2, $m$ is stored as an integer value in the {\tt gex} element in the File
Descriptor, multiplied by a factor 10:
\begin{equation}
  \label{equ:gex}
  gex = \mathrm{nint} (10 \times m)
\end{equation}
This means e.g. that equal-size extensions use $gex = 10$ and a 2-based
exponential growth uses $gex = 20$.

\subsection{The Extensions}
\label{txt:extensions}

Each extension {\tt iex} contains a set of entries, up to $lex_E(iex) =
lex1 \times \left(\frac{gex}{10}\right)^{iex-1}$ entries, according to the
equations \ref{equ:lexe} and \ref{equ:gex}. It is divided in 2 main parts:
\begin{enumerate}
\item the {\it Extension Index},
\item the {\it File Entries}.
\end{enumerate}
When an extension is full (\ie\ the number of entries has reached its
limit), new file entries are added to a new extension.

\subsubsection{The Extension Index}
\label{txt:extensionindex}

Each entry is basically described by a few elements in its associated {\it
  Entry Index}. Its length is given by the {\it Index Length} element in
the File Descriptor ({\tt lind}). The interpretation of the bytes of the
{\it Entry Index} is delegated to the application, with one restriction:
the application must provide a way to find the entry in the file. This is
achieved thanks to the 2 first elements in the entry index, which must give
the entry address in the file, \ie\ its starting record (absolute position
in the file, coded as a Fortran Integer*8) and word (relative from the
beginning of this record, coded as a Fortran Integer*4). Since the
interpretation of most of the {\it Entry Index} is delegated to the
application, the \cdc{} provides an Index Version parameter ({\tt vind}).
The application can use its own versioning with the following restriction:
the Index Version must be greater than 1 in File Version 2\footnote{In
  other words: Index V1 in File V1, Index V2 or more in File V2. Index V1
  can not be used in File V2 since at least the first word (Entry Address)
  is different.}. The couple {\tt vind + lind} can be used as desired by
the application to update and ensure backward support of all the revisions
of the Entry Index.

The Extension Index is the collection of the entry indexes of the file
entries gathered in the extension. It always starts at the beginning of a
record.  The {\tt lind} words are repeated contiguously, in the limit of
$lex_E(iex)$ entry indexes. If a non-integer number of indexes fit in one
record ($reclen/lind \notin \mathbb{N}$), entry indexes can overlap two
records. The last bytes of the last record in the extension index may be
left empty if the last entry index does not reach the end of the record.
This mathematically translates into a \texttt{ceiling} statement in
Eq.~\ref{equ:lexr}. Space is always provisioned (the $lex_R(iex)$ records)
for the $lex_E(iex)$ entries description, even though the associated file
entries are not yet present.

\subsubsection{The File Entries}

The data written in each entry is very variable, \eg{} spectra may have
very different number of channels.
%This means that the length of
%each entry is variable. 
As a consequence, the starting address of an entry cannot be computed from the
file entry number and is stored in the Entry Index.
%There is then a mechanism which describes
%which memory space is reserved, and where it is reserved. 
%An entry starts
%at the address found in its entry index.
An entry does not necessarily start and end on record boundaries. It is
divided into
\begin{enumerate}
\item the {\it Entry Descriptor},
\item the {\it Observation} itself.
\end{enumerate}

\paragraph{The Entry Descriptor}
\label{txt:obsind}

\TabEntryDesc{} %

The Entry Descriptor gives a technical description (version of the
observation, memory layout, ...) of the associated observation in the Entry
(see Table~\ref{tbl:entdesc}). Except the {\tt code} identifier, all these
elements can be different from one entry to an other:
\begin{enumerate}
\item the {\tt code} is a unique identifier used to check the file
  consistency. It is always set to the 4 characters {\tt
    `2$\sqcup\sqcup\sqcup$'} (number 2 followed by 3 blanks).
\item the {\tt version} can be used by the application to keep track of the
  evolutions of the content of the sections and/or data array.
\item {\tt nsec} describes the number of sections actually written in the
  current descriptor.
\item {\tt nword} is the actual number of words of the whole entry, this
  descriptor included. It is used to prevent overflow at reading/writing time.
\item {\tt adata} is the data address, in words from the beginning of the
  descriptor.
\item {\tt ldata} is the data length, in words (4-byte words whatever the
  kind and precision of data actually stored).
\item {\tt xnum} is the entry number, available here for
  self-consistency of the file (\ie\ the entry being $N^{th}$ in the file
  index must have {\tt xnum} $= N$).
\item {\tt seciden} is an array of {\tt nsec} elements providing the
  identifiers (Integer*4 codes, private to the application) of the sections
  present in the observation.
\item {\tt secleng} is an array of {\tt nsec} elements providing the length
  of each of these sections.
\item {\tt secaddr} is an array of {\tt nsec} elements providing the
  addresses of each of these sections, in words from the beginning of the
  entry.
\end{enumerate}

{\tt msec} is the maximum number of sections which can be written in a
given entry (while {\tt nsec} is the actual number of sections which are
written in this entry). {\tt msec} is let to the choice of the application.
It is not stored in the descriptor, but makes sense at the time each new
section is written and the descriptor is filled accordingly. If the
application writes less {\tt nsec} than {\tt msec}, the remaining space in
the descriptor is left empty. This can lead to non-negligible byte losses
(see Section~\ref{txt:loss}). As {\tt nsec}, {\tt msec} may vary from entry
to entry in the same file.

\paragraph{The Observation}

The science measurements associated to each observation are shared between
\begin{itemize}
\item the {\it Observation Header}. Each Observation Header is divided
  in several sections, containing scalar values or arrays. They are
  defined and controled by the calling application. Each type of
  section is coded with an integer code, known only by the
  application. It is not required that all sections be present. The
  space available for the section elements is reserved at first
  \emph{write} time ({\tt secleng(i)}). One or more \emph{update}s of
  the section, including on its number of elements, are possible, as
  long as they keep the length smaller or equal than the reserved
  space. Enlarging a section during an update is impossible and is
  then forbidden\footnote{If enlarging of a section is required, the
    application or the user can write a new version of the observation
    in the same file (\emph{multiple} files) or in a new file.}. 
\item and usually (but not necessarily) {\it Data}, e.g., a spectrum coded
  as a single floating point array of intensities.
\end{itemize}
The final user will probably have access only to these information, all
other information described in this document remaining hidden. Sections and
Data can be mixed in any order. They are not necessarily aligned on the
file records. Their address can be retrieved from the Entry Descriptor. The
first element always starts after the entry descriptor, \ie\ at the address
$11+5 \times msec+1$, in words from the beginning of the entry. This
structure makes observations self-contained: all the associated information
is recorded on a few contiguous file records.

%%%%%%%%%%%%%%%%%%%%
\section{Storage efficiency}
\label{txt:loss}

The \cdc{} is defined such as it introduces unused bytes here and there.
We paid special attention to reduce this to a negligeable fraction of the
file. In particular, the Version 1 of the \cdc\ had an important source of
unused bytes, \ie{} the fact that each entry had to start at the beginning
of a record. Depending on the record length in comparison to the entry
size, the number of lost words goes from 0 to {\tt reclen}-1. Which can
lead to terrible results (e.g. with $1 + \epsilon$ records actually used
for 2 allocated, the mathematical limit is 50\% of the file unused). This
has been addressed in the Version 2: Each entry can now start anywhere in a
record, \ie{}, right after the previous entry. The key point to keep in
mind in this discussion is that all of these unused bytes will still be
read (at least by the \clib{} implementation because of the record per
record access to the file). This means basically that if 10\% of the file
is unused, 10\% of the time is spent for useless I/O.

This section is a exhaustive account of the storage efficiency, in \cdc{}
version 2.
\begin{enumerate}
\item The only lost bytes in the File Descriptor are associated to the
  unused extension addresses. Their impact is low.
\item The File Index has several sources of bytes loss. The first is the
  unused bytes in the Entry Index itself, but this is under the control of
  the application. Note that this value is proportional to the number of
  entries in the file.
\item The second element in the Index is the provisioned but unused bytes
  in the last extension index. This point explains why a file with a few
  entries can be very large (depending on the {\tt lex1} scaling factor).
\item The third source of loss has been added by the Version 2 format.
  Since a non-integer number of entry indexes can fit in one record, there
  can be lost words at the end of the last record devoted to each extension
  indexes. However this remains small (at most $lind-1$ words per
  extension).
\item In the Entry Descriptor, space for up to {\tt msec} sections has to
  be pre-allocated while only {\tt nsec} will be used. The effect can
  become important because it is proportional to the number of entries in
  the file. However this particular point is under the control of the
  calling application. This application always can dimension the Entry
  Descriptor to its exact useful size.
\end{enumerate}

%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
\section{Basic file reading example}

Given the rules and elements described in the previous section, we list
here how we can find, for example, the first section and the data array of
the $ient^{th}$ entry.
\begin{enumerate}
\item check the first word of the file: make sure that file code is one of
  those described in Table~\ref{tbl:classiccodes}, and if you will need to
  convert all the values from one bit encoding system to another. Hereafter
  we assume you are reading a version 2 file.
\item the second word provides the record length (in words). Many
  quantities (\eg\ addresses) are provided in the \cdc{} in number of
  records of this length.
\item with the third word, the application can check that it reads a file
  it understands.
\item read {\tt lind}, the entry index length, in the 5$^{th}$
  word. It will be used below.
\item then read {\tt lex1} and {\tt gex} in the 12$^{th}$ and
  14$^{th}$ words. They give the number of entries in the first
  extension, and the rule for the number of entries in the subsequent
  extensions. Resolving Eq.~\ref{equ:nentry} you should be able to
  find in which extension the $ient^{th}$ entry is (say $iex^{th}$),
  and the relative number of this entry in its extension (say
  $jent(iex)$).
\item find where this extension should be found. Its address is available
  in the two words starting at $13+2 \times iex$ in the file descriptor.
\item knowing the relative number $jent$ of the entry in its extension
  $iex$, you can find in which relative record (from the beginning of
  the extension index) the entry index is ($((jent-1)*lind)/reclen$),
  and where the starting word in this record
  ($\mathrm{mod}((jent-1)*lind,reclen)+1$).
\item read the {\tt lind} words of the entry index. The first 3 words
  provide the entry address: the record (first and second words) and
  starting word (third word). The others words are application
  specific.
\item read the given record, at the given position. The first words are the
  entry descriptor, as detailed in Table~\ref{tbl:entdesc}.
\item then read the number of section ({\tt nsec}) defined in the
  descriptor. The length of the first section is found at the words 12-13,
  and its address at $12+2 \times nsec$ and next word. Now you can read
  them, taking also care of possible records overlap.
\item in order to give a signification to these bytes, get the section
  identifier in the word $12+4 \times nsec$ and refer to the
  documentation of the application which wrote them. Congratulations,
  you've just read the first section!
\item The data address ({\tt adata}, from the beginning of the entry)
  and length ({\tt ldata}), both in number of words, are encoded as
  two Integer*8 from words 6 to 9 in the Entry Descriptor. Just read
  the {\tt ldata} words starting from {\tt adata}, taking care that
  the data array can overlap several records. Now you have just read
  the data array.
\end{enumerate}

\emph{Acknowledgments}. The authors thank S. Guilloteau, R. Lucas and
T. Forveille for useful comments about this document.

\clearpage


%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
\appendix

%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
\section{The \cdc{} Version 1}
\label{txt:cdcv1}

This appendix briefly describes the historical version of the \cdc.

\begin{table}[htb]
\begin{center}
\caption{File Descriptor Version 1}
\vspace{1ex}
\begin{tabular}{l|cl|l}
Position           & Parameter    & Fortran Kind & Purpose \\
\hline
  1                & {\tt code}   & Character*4  & File code                           \\
  2                & {\tt next}   & Integer*4    & Next free record                    \\
  3                & {\tt lex}    & Integer*4    & Length of first extension (number of entries) \\
  4                & {\tt nex}    & Integer*4    & Number of extensions                \\
  5                & {\tt xnext}  & Integer*4    & Next available entry number         \\
  6:2*{\tt reclen} & {\tt ex(:)}  & Integer*4    & Array of extension addresses
\end{tabular}
\end{center}
\end{table}

Note the following remarks about the File Descriptor in the Version 1:
\begin{enumerate}
\item the File Code was starting with the letter `1' or `9' (see
  Table~\ref{tbl:classiccodes}).
\item the record length was always set to 128 words. This proved to be a
  limiting factor in the I/O accesses, especially for data acquisition at
  the telescope where the system buffering is disabled for the real-time
  processing purpose.
\item there was no File Kind. The reading application had to open the
  observations one by one and accept or reject them depending of their
  kind, assuming the section providing this information could be read from
  any program.
\item it was difficult to update the Index (no versioning). Its length was
  always 32 words, some of them unused.
\item the {\it single} or {\it multiple} attribute was encoded in the File
  Code.
\item since {\tt next} was encoded as a Fortran Integer*4, it was
  impossible to write a file larger than $2^{31}-1$ records ($\sim$ 1 TB).
\item file entries always started at the beginning of the {\tt next}
  available record, possibly leaving unused space in the previous record.
\item all extensions had the same length ({\tt lex}). This made impossible
  to have a file at the same time i) dimensioned to accept a large number
  of entries, and ii) remaining small if storing a few entries (because of
  the Index provision in each extension).
\item the {\tt xnext} parameter was encoded as a Fortran Integer*4, which
  limited the maximum number of entries per file to $2^{31}-1$.
\item the extension addresses were spreading over the 2 first records,
  which always implied at most 251 extensions.
\end{enumerate}

Regarding the File Index in Version 1 Files, not much is to be said since
it is not strictly under the control of the \cdc. However, here also the
first word of the Index is expected to be the Entry address (in absolute
records), but encoded as a Fortran Integer*4. Note also that since its
length was always 32 words, 4 of them fitted exactly in one record (always
128 words).

\begin{table}[htb]
\begin{center}
\caption{Entry Descriptor Version 1}
\vspace{1ex}
\begin{tabular}{lcl|l}
Position      & Parameter             & Fortran Kind & Purpose \\
\hline
  1           & {\tt code}            & Character*4  & Identifier of Entry Descriptors                \\
  2           & {\tt nbloc}           & Integer*4    & Number of records of this entry                \\
  3           & {\tt nword}           & Integer*4    & Number of words of this entry                  \\
  4           & (unused)              &              &                                                \\
  5           & {\tt adata}           & Integer*4    & Data address (in words) from the beginning of the entry \\
  6           & {\tt ldata}           & Integer*4    & Data length (in words)                         \\
  7           & (unused)              &              &                                                \\
  8           & {\tt nsec}            & Integer*4    & Number of section                              \\
  9           & {\tt xnum}            & Integer*4    & Entry number                                   \\
 10:...       & {\tt seciden(1:nsec)} & Integer*4    & Section lengths (in words)                     \\
 ...:...      & {\tt secleng(1:nsec)} & Integer*4    & Section addresses (in words)                   \\
 ...:9+3*nsec & {\tt secaddr(1:nsec)} & Integer*4    & Section numbers (identifiers)                  \\
 ...:9+3*msec & (unused)              &              & Empty room for up to 'msec' sections
\end{tabular}
\label{tbl:entdescv1}
\end{center}
\end{table}

In the Entry Descriptor Version 2, only the number of records used by the
entry ({\tt nbloc}, redundant with {\tt nword}) and the unused words have
been removed in comparison to the Version 1 (see
Table~\ref{tbl:entdescv1}). In version 1, the various addresses and
measurements parameters were coded as Fortran Integer*4, which basically
limited the Observation size to $2^{31}-1$ words ($\sim$ 8 GB).

\begin{table}[htb]
\begin{center}
\caption{\cdc{} Version 1 limits. Most of them are the limit
  an IEEE Fortran Integer*4 provides ($2^{31}-1 = 2147483647$).}
\vspace{1ex}
\begin{tabular}{l|rl|l}
Parameter         & Limit      & Unit                   & Comment \\
\hline
{\tt next}        & 2147483647 & records per file       & Maximum 1 TB for the whole file \\
{\tt lex}         & 2147483647 & entries per extension  &  \\
{\tt nex}         &        251 & extensions per file    &  \\
{\tt xnext}       & 2147483647 & entries per file       &  \\
{\tt ex(:)}       & 2147483647 & record                 & Absolute extension addresses in file \\
\hline
{\tt nbloc}       & 2147483647 & records per entry      & Maximum 1 TB for one observation \\
{\tt nword}       & 2147483647 & words per entry        & Maximum 8 GB for one observation \\
{\tt adata}       & 2147483647 & word                   & Relative data address in the entry \\
{\tt ldata}       & 2147483647 & data values            & e.g. maximum number of channels \\
{\tt secaddr(:)}  & 2147483647 & word                   & Relative section address in the entry
\end{tabular}
\label{tbl:cdclimits}
\end{center}
\end{table}

The Table~\ref{tbl:cdclimits} summarizes the internal limits of the \cdc{}
Version 1. All these limits could be gathered into 3 main categories
\begin{itemize}
\item the limit on the file size. The more restrictive is the number of
  records per file: 2147483647 records = 1 TB.
\item the limit on the number of observations per file: at most 2147483647.
\item the limit on the observation size, ceiled by the physical size of the
  data: 2147483647 words = 8 GB.
\end{itemize}
The Version 2 of the \cdc{} standard have addressed all these limitations,
by turning the corresponding parameters into Fortran Integer*8 values.

\end{document}

%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%

