% This is LLNCS.DEM the demonstration file of
% the LaTeX macro package from Springer-Verlag
% for Lecture Notes in Computer Science,
% version 2.2 for LaTeX2e
%
\documentclass{llncs}
%
\usepackage{makeidx}  % allows for indexgeneration
\usepackage{graphicx,psfig,epsfig}
%
\begin{document}
%
%
%
%
%
% Snip snap scissors go, snip snap ... 
%
%
\title{Recognition and Labeling of Prosodic Events in Slovenian Speech}
%
%\titlerunning{Hamiltonian Mechanics}  % abbreviated title (for running head)
%                                     also used for the TOC unless
%                                     \toctitle is used
%
\author{France Miheli\v c\inst{1} \and  Jerneja Gros\inst{1} \and Elmar N\"oth\inst{2} \and
      Volker Warnke\inst{2}}
%
%\authorrunning{France Miheli\v c et al.}   % abbreviated author list (for running head)
%
%%%% modified list of authors for the TOC (add the affiliations)
\tocauthor {France Miheli\v c, Jerneja Gros (University of Ljubljana)}
%
\institute{University of Ljubljana, Faculty od Electrical Engineering,\\ 
           Tr\v za\v ska 25, SI-1000 Ljubljana, Slovenia\\
           \email{\{mihelicf, nejka\}@fe.uni-lj.si},\\ WWW home page:
           \texttt{http://luz.fe.uni-lj.si}
\and
	   Universit\"at Erlangen-N\"urnberg, \\Martensstrasse 3, 91058 Erlangen, BRD\\
           \email{\{noeth, warnke\}@informatik.uni-erlangen.de}}

\maketitle              % typeset the title of the contribution

\begin{abstract}
The paper describes prosodic annotation procedures of the GOPOLIS
Slovenian speech data database and methods for automatic classification
of different prosodic events.
Several statistical parameters concerning duration and loudness of words,
syllables and allophones were computed for the Slovenian language,
for the first time on such a large amount of speech data.
The evaluation of the annotated data
showed a close match between automatically determined
syn\-tac\-tic-prosodic boundary mar\-ker positions and those obtained by
a rule-based approach.
\end{abstract}


%**********************
\section{Introduction}
Research and development
in the field of spoken language technology
encouraged and supported by the European Community resulted in many
experimental and also commercially available systems for speech recognition,
understanding, synthesis and dialogue for different European languages
\cite{And98}.

Noticeable progress in this field in the recent years
was also made for one of the minor European languages, Slovenian.
As a result of our own investigations and our
collaboration with foreign
partners, a multilingual dialogue system was developed \cite{ICSLP98}.
Slovenian speech recognition, understanding and synthesis systems are
the most important achievements of the research group in Slovenia \cite{Ivo99}.

The usage of prosodic parameters in speech recognition and understanding
and quality prosody modeling in speech synthesis resulted in large improvements
in the performance of these systems \cite{SC98,BO98,Gros98}.
At present not much work has been done in this field for the Slovenian
language. The research group at the University Erlangen-N\"urenberg has some
important research results in the field of prosodic analysis for the German
language \cite{SC98,Kiss97,Kompe97} and also some experience with  prosody
processing for other languages \cite{ErES99}. In a combined effort of the
Slovenian and the German research groups procedures for automatic prosodic
parameter measurement and evaluation
were tested on Slovenian speech material. We tried to recognize prosodic
events and evaluate them statistically.

\section{Selection of prosodic events}
Syntactic-prosodic boundaries
along the lines of \cite{SC98},
annotated for transliterations of read speech
and acoustic-prosodic boundaries and word accents labeled via acoustic
perceptual sessions were chosen for the initial experiments.
\begin{table}[htbp]
\begin{center}
\begin{itemize}
\item {\sf M3}: clause boundaries
\item {\sf M2}: constituent boundaries likely to be marked prosodically
\item {\sf M1}: boundaries that syntactically belong to the normal constituent
    boundaries probably not marked prosodically
    because they are "close" to a {\sf M3} boundary
\item {\sf M0}: every other word boundary
\end{itemize}
\end{center}
\caption{Syntactic-prosodic boundary labels} \label{SPB}
\end{table}

\vspace{-2mm}
We performed automatic labeling
with syn\-tac\-tic-pro\-so\-dic boundaries of the Slovenian {\sc Gopolis} speech
corpus (50 speakers, 8.645 utterances, 5.077 different corpus sentences)
according to \cite{Kompe97} pp. 140--144.
In the labelling procedure, we
distinguished between 4 types of
boundaries as listed in Table ~\ref{SPB}.
Here is example of labeled text\footnote{
{\sc M0} labels are not indicated in the example.}:

\vspace{1mm}\noindent
{\em
Ali {\sf M1} imate {\sf M2} kak\v{s}no letalo {\sf M3} \v{c}ez tri tedne?\\
(Is there a flight in three weeks?)}

\vspace{1mm}\noindent
Acoustic-prosodic boundaries and word accent labels were defined as in
{\sc Verbmobil} \cite{Kompe97}. They are described in
Tables \ref{APB} and \ref{WA}.
\begin{table}[htbp]
\begin{center}
\begin{itemize}
\item {\sf B3}: prosodic clause boundary
\item {\sf B2}: prosodic phrase boundary
\item {\sf B9}: irregular boundary, usually hesitation lengthening
\item {\sf B0}: every other word boundary
\end{itemize}
\end{center}
\caption{Acoustic-prosodic boundary labels} \label{APB}
\end{table}

\vspace{-6mm}
\begin{table}[htbp]
\begin{center}
\begin{itemize}
\item {\sf PA}: the most prominent (primary) accent within a prosodic clause
\item {\sf SA}: all other accented words are marked as carrying secondary accent
%"NA" replaced by "SA"
\item {\sf UA}: unaccented words
\end{itemize}
\end{center}
\caption{Word accent labels} \label{WA}
\end{table}
\noindent
Here is a  labeled sample sentence\footnote{
The default classes
{\sc B0} and {\sf UA} labels are not indicated in the example.}:

\noindent
{\em
Lahko ponovite {\sf SA} odgovor {\sf PA} , {\sf B3} prosim {\sf PA} ?\\
(Can you repeat the answer, please?)}


\section{Data preparation}
The experiments on prosodic events classification we wanted to perform required
some specific
data preparation. Speech signals of read Slovenian texts from the
{\sc Gopolis} speech
corpus \cite{GOPO98} were used for the experiments. The {\sc Gopolis} pronunciation
dictionary was extended  with
stress and syllable markers. Special categories denoting
{\em silence, non-word, consonants} and {\em syllable-root} were added.
A special format for representing word graphs was used.

The segmentation of the speech signals was
performed automatically using the {\sc Isadora} net \cite{Sch95}.
An important feature of the Isadora net environment is the automatic
detection of silence segments not indicated in the transliteration.

The text corpus of the {\sc Gopolis} database has been created automatically using a
context-free grammar consisting of 189 sentence templates \cite{Nejka95}. The
sentence templates cover the most frequent dialogue situations
occurring at airline
timetable information retrieval.

In order to obtain a version of the {\sc Gopolis} database
annotated with boundary information, {\sf M1, M2}
and {\sf M3} labels were inserted into the sentence templates. Then the sentence
generation process was repeated.

The {\sf M2} markers were generally set according to rhythmic constraints.
{\sf M1} markers --- as described above ---
were set, if an {\sf M2} boundary was too close to the beginning or end
of the utterance or too close to an {\sf M3} boundary,
i.e. they were also set according to rhythmic constraints.
{\sf M3} markers
were placed between main/subordinate clauses and around embedded clauses.

Acoustic-prosodic boundaries and word accent labels were marked
manually by perception tests using visualisation of speech
signals and listening. In total, 1000 signals (6 speakers)
were labeled, representing 12\% of the whole {\sc Gopolis} speech corpus.
The annotation was conducted by the first author;
intralabeller consistency has not yet been evaluated.


\section{Features selection and pattern classification}
Classification using only prosodic feature sets describing
duration segmental characteristics on the word level, speaking rate, energy
and pitch was performed.
%EN OK
Pitch periods were computed for the entire {\sc Gopolis} database based on the
inverse filtering algorithm \cite{Kiss97}.
Various statistical parameters previously determined on the training
set were used in the duration and energy normalization procedures \cite{Kiss97}
(pp. 165--182).
%EN OK
All 95 features forming "the best word features set" according to experiments
performed on German speech \cite{Kiss97} (page 258) were computed for the
Slovenian speech data.

Energy and pitch features are based on the short-term
energy and F0 contour, respectively. Some of the features that are used to
describe a pitch contour are shown in Figure \ref{FIGfeatureex1}.
Additionally, we use the mean and the median as features \cite{Buc99}.

Neural nets were used for classification. The SNNS software \cite{SNNS} was
used for neural net modeling. Several net configurations and initializations
were tested in the experiments.

\begin{center}
  \begin{figure}
    \vspace*{-2ex}
    \centerline{\psfig{figure=features.eps,width=0.70\textwidth }}
    \caption{Example of features used to describe a pitch contour.}
    \label{FIGfeatureex1}
    \vspace*{-3ex}
  \end{figure}
\end{center}

%%WA END INSERTION

%EN OK
%Some results yielding the best recognition rates achieved on test
%sets are presented in next section.


\section{Recognition results}
Four classification schemes
with different clustering for the syn\-tac\-tic-prosodic boundaries 
were tested.
The results indicating overall recognition rate and classwise
computed recognition rate are displayed
%EN OK overall rec. rate and classwise computed rec. rate
%EN OK which is which?
in Table \ref{RecSPB}. 5 female and 5 male speakers were selected for the
test set while the remaining 40 speakers were used in the training set.

The classification of {\sf M3} markers versus other categories and
classification into three classes ({\sf M3 : M2 : M1, M0}) are the most
important for further use in semantic parsing of recognized word strings
\cite{BO98}. By comparing the obtained results
to the results obtained on the {\sc Erba} corpus \cite{Erba94} (82\%, 72\%), the results
on Slovenian speech data are even slightly better. 
This indicates at least the correctness
of the automatic procedure for boundary labeling and also the appropriate
feature selection and classification procedure for the Slovenian language.

%EN OK This is hard to understand. I would erase it
%OK With a combined classification of syntactic-prosodic boundaries and word accents
%OK on the German speech data were further improved. (90\% for 3-class problem)
%OK \cite{Kiss97}.

\begin{table}[htbp]
\begin{center}
\begin{tabular}{|c|c|c|}
\hline
Classification & Recognition   & Recognition \\
schemes        &  overall      & classwise \\
\hline
{\sf M0 : M1 : M2 : M3} & 74.1\% & 77.5\% \\
\hline
{\sf M0 M1 : M2 : M3} & 80.7\% & 80.3\% \\
\hline
{\sf M0 M1 M2 : M3} & 87.3\% & 87.7\% \\
\hline
{\sf M0 : M1 M2 M3} & 87.5\% &  87.6\% \\
\hline
\end{tabular}
\end{center}
\caption{Recognition results for different classification schemes for
syntactic-prosodic boundaries.
App. 40000 labels were used in the training set, 8400 in the test set,
label {\sf M3} at the end of the utterance was not included
in the tests.}
\label{RecSPB}
\end{table}

Due to the lack of manually labeled data\footnote{Only approx. 12\% of all
{\sc Gopolis} corpus was manually labeled.} the results of the experiments on
acoustic-prosodic
boundary classification and word accent classification are not so reliable.
They might be significantly improved using a larger training set. The results 
are displayed in Table \ref{RecAPBWA}.
Two turns of classification experiments were performed.
Both times 5 speakers were used for training and 1 speaker
for testing.

\begin{table}[htbp]
\begin{center}
\begin{tabular}{|c|c|c|c|}
\hline
Speaker & Classification & Recogn.      & Recogn. \\
        & schemes        & overall          & classwise \\
\hline
01M & {\sf B0 : B9 : B2 : B3} & 73.9\% & 79.8\% \\
\hline
03M & {\sf B0 : B9 : B2 : B3} & 66.8\% & 67.2\% \\
\hline \hline
01M & {\sf UA : SA : PA} & 64.1\% & 62.5\% \\
\hline
03M & {\sf UA : SA : PA} & 72.5\% & 69.9\% \\
\hline
\end{tabular}
\end{center}
\caption{Recognition results for different classification schemes for
acoustic-prosodic boundaries and word accent labels.
Approx. 4500 labels were used in the training set and 840 labels in the
test set for acoustic-prosodic boundaries; the label {\sf B3}
at the end of the utterance was not included in the tests.
Approx. 5000 labels in the training set and 1000 labels
in the test set for the word accent labels.}
\label{RecAPBWA}
\end{table}

\section{Conclusion}
Useful knowledge on prosodic events for the
Slovenian speech was gained as an intermediate results of the described experiments.
Pitch periods for the complete {\sc Gopolis} speech data base were computed and
they can be used as a reference for further investigations. Several statistical
parameters concerning duration and loudness of words, syllables and allophones
were computed for the Slovenian language, for the first time on such a large
speech database.

The obtained research results encourage us to continue the co-operation of both
partner groups.
Data collection and annotation as well as development of semantic parsers,
using prosodic information as additional input, will be the next steps of our
research. The computed prosodic parameters will also be applied in the
prosody prediction process for Slovenian text-to-speech synthesis.

\section{Acknowledgement}
France Miheli\v c thanks the DAAD for its support enabling his study visit
in Erlangen from May to July 1999, during which most of the presented
work was done. Special thanks go to F. Gallwitz and J. Buckow who
helped organizing the experiments in Erlangen and to
S. Dobri\v sek and J. \v Zibert from the University of Ljubljana for their support.

\bibliography{lrec1}

\end{document}


