INN commit: trunk (37 files)

INN Commit rra at isc.org
Fri Dec 25 09:58:00 UTC 2020


    Date: Friday, December 25, 2020 @ 01:57:59
  Author: iulius
Revision: 10469

Initial implementation of the ovsqlite overview storage method

Celebration!
Bo Lindbergh has implemented a new overview storage method based
on SQLite, known for its long-term stability and compatibility.
A perfect choice to store overview data!

This is the original implementation for INN 2.7.0.

The new value "ovsqlite" can be used as an "ovmethod" in inn.conf.

Many thanks again, Bo!

Added:
  trunk/doc/pod/ovsqlite-server.pod
  trunk/doc/pod/ovsqlite.pod
  trunk/m4/sqlite3.m4
  trunk/samples/ovsqlite.conf
  trunk/storage/ovsqlite/
  trunk/storage/ovsqlite/ovmethod.config
  trunk/storage/ovsqlite/ovmethod.mk
  trunk/storage/ovsqlite/ovsqlite-private.c
  trunk/storage/ovsqlite/ovsqlite-private.h
  trunk/storage/ovsqlite/ovsqlite-server.c
  trunk/storage/ovsqlite/ovsqlite.c
  trunk/storage/ovsqlite/ovsqlite.h
  trunk/storage/ovsqlite/sql-init.sql
  trunk/storage/ovsqlite/sql-main.sql
  trunk/storage/ovsqlite/sqlite-helper-gen.in
  trunk/storage/ovsqlite/sqlite-helper.c
  trunk/storage/ovsqlite/sqlite-helper.h
Modified:
  trunk/MANIFEST
  trunk/Makefile.global.in
  trunk/configure.ac
  trunk/doc/FAQ
  trunk/doc/man/	(properties)
  trunk/doc/man/Makefile
  trunk/doc/pod/Makefile
  trunk/doc/pod/checklist.pod
  trunk/doc/pod/inn.conf.pod
  trunk/doc/pod/install.pod
  trunk/doc/pod/makehistory.pod
  trunk/doc/pod/news.pod
  trunk/doc/pod/rc.news.pod
  trunk/scripts/rc.news.in
  trunk/site/	(properties)
  trunk/site/Makefile
  trunk/storage/Makefile
  trunk/support/getrra-c-util
  trunk/support/mkmanifest
  trunk/tests/Makefile

---------------------------------------+
 MANIFEST                              |   23 
 Makefile.global.in                    |    9 
 configure.ac                          |    4 
 doc/FAQ                               |   18 
 doc/man                               |    2 
 doc/man/Makefile                      |    8 
 doc/pod/Makefile                      |   11 
 doc/pod/checklist.pod                 |    2 
 doc/pod/inn.conf.pod                  |   17 
 doc/pod/install.pod                   |   45 
 doc/pod/makehistory.pod               |   55 
 doc/pod/news.pod                      |   19 
 doc/pod/ovsqlite-server.pod           |   75 +
 doc/pod/ovsqlite.pod                  |  107 +
 doc/pod/rc.news.pod                   |    8 
 m4/sqlite3.m4                         |   87 +
 samples/ovsqlite.conf                 |   46 
 scripts/rc.news.in                    |   60 
 site                                  |    1 
 site/Makefile                         |    9 
 storage/Makefile                      |   61 
 storage/ovsqlite                      |    6 
 storage/ovsqlite/ovmethod.config      |    7 
 storage/ovsqlite/ovmethod.mk          |   19 
 storage/ovsqlite/ovsqlite-private.c   |   65 
 storage/ovsqlite/ovsqlite-private.h   |  484 +++++++
 storage/ovsqlite/ovsqlite-server.c    | 2174 ++++++++++++++++++++++++++++++++
 storage/ovsqlite/ovsqlite.c           | 1107 ++++++++++++++++
 storage/ovsqlite/ovsqlite.h           |   69 +
 storage/ovsqlite/sql-init.sql         |  117 +
 storage/ovsqlite/sql-main.sql         |  203 ++
 storage/ovsqlite/sqlite-helper-gen.in |  209 +++
 storage/ovsqlite/sqlite-helper.c      |  101 +
 storage/ovsqlite/sqlite-helper.h      |   55 
 support/getrra-c-util                 |   32 
 support/mkmanifest                    |    5 
 tests/Makefile                        |    2 
 37 files changed, 5264 insertions(+), 58 deletions(-)

Modified: MANIFEST
===================================================================
--- MANIFEST	2020-12-25 09:51:06 UTC (rev 10468)
+++ MANIFEST	2020-12-25 09:57:59 UTC (rev 10469)
@@ -200,6 +200,8 @@
 doc/man/ovdb_server.8                 Manpage for ovdb_server
 doc/man/ovdb_stat.8                   Manpage for ovdb_stat
 doc/man/overchan.8                    Manpage for overchan backend
+doc/man/ovsqlite-server.8             Manpage for ovsqlite-server
+doc/man/ovsqlite.5                    Manpage for the ovsqlite overview module
 doc/man/passwd.nntp.5                 Manpage for passwd.nntp config file
 doc/man/perl-nocem.8                  Manpage for perl-nocem
 doc/man/pgpverify.1                   Manpage for pgpverify
@@ -305,6 +307,8 @@
 doc/pod/ovdb_server.pod               Master file for ovdb_server.8
 doc/pod/ovdb_stat.pod                 Master file for ovdb_stat.8
 doc/pod/overchan.pod                  Master file for overchan.8
+doc/pod/ovsqlite-server.pod           Master file for ovsqlite-server.8
+doc/pod/ovsqlite.pod                  Master file for ovsqlite.5
 doc/pod/passwd.nntp.pod               Master file for passwd.nntp.5
 doc/pod/procbatch.pod                 Master file for procbatch.8
 doc/pod/prunehistory.pod              Master file for prunehistory.8
@@ -595,6 +599,7 @@
 m4/snprintf.m4                        Autoconf macro to check snprintf
 m4/socket-unix.m4                     Autoconf macros to check UNIX domain sockets
 m4/socket.m4                          Autoconf macros to check socket support
+m4/sqlite3.m4                         Autoconf macros for SQLite v3 support
 m4/syslog.m4                          Autoconf macro for syslog facility
 m4/users.m4                           Autoconf macro for INN users
 m4/vamacros.m4                        Autoconf macro to check variadic macros
@@ -677,6 +682,7 @@
 samples/nntpsend.ctl                  Outgoing nntpsend feed configuration
 samples/nocem.ctl                     Config file for perl-nocem
 samples/ovdb.conf                     Berkeley DB overview configuration
+samples/ovsqlite.conf                 SQLite overview configuration
 samples/passwd.nntp                   Passwords for remote connections
 samples/readers.conf                  Reader connection configuration
 samples/send-uucp.cf                  send-uucp configuration file
@@ -736,6 +742,23 @@
 storage/ovinterface.h                 Overview API interface
 storage/ovmethods.c                   Generated table of overview methods
 storage/ovmethods.h                   Generated interface to overview methods
+storage/ovsqlite                      ovsqlite overview method (Directory)
+storage/ovsqlite/ovmethod.config      buildconfig definitions for ovsqlite
+storage/ovsqlite/ovmethod.mk          Make rules for ovsqlite
+storage/ovsqlite/ovsqlite-private.c   Private code for ovsqlite
+storage/ovsqlite/ovsqlite-private.h   Private header for ovsqlite
+storage/ovsqlite/ovsqlite-server.c    SQLite database exclusive owner
+storage/ovsqlite/ovsqlite.c           ovsqlite implementation
+storage/ovsqlite/ovsqlite.h           ovsqlite interface
+storage/ovsqlite/sql-init.c           Generated database setup implementation
+storage/ovsqlite/sql-init.h           Generated database setup interface
+storage/ovsqlite/sql-init.sql         SQLite code for database setup
+storage/ovsqlite/sql-main.c           Generated daily operation implementation
+storage/ovsqlite/sql-main.h           Generated daily operation interface
+storage/ovsqlite/sql-main.sql         SQLite code for daily operation
+storage/ovsqlite/sqlite-helper-gen.in Package SQLite code for convenient use
+storage/ovsqlite/sqlite-helper.c      SQLite code package implementation
+storage/ovsqlite/sqlite-helper.h      SQLite code package interface
 storage/timecaf                       timecaf storage method (Directory)
 storage/timecaf/README.CAF            README the CAF file format
 storage/timecaf/caf.c                 CAF file implementation

Modified: Makefile.global.in
===================================================================
--- Makefile.global.in	2020-12-25 09:51:06 UTC (rev 10468)
+++ Makefile.global.in	2020-12-25 09:57:59 UTC (rev 10469)
@@ -86,11 +86,18 @@
 
 ##  Berkeley DB support.  If this support is configured, anything linking
 ##  against libstorage also needs to link against BDB_LDFLAGS and BDB_LIBS.
- 
+
 BDB_CPPFLAGS   = @BDB_CPPFLAGS@ $(ZLIB_CPPFLAGS)
 BDB_LDFLAGS    = @BDB_LDFLAGS@ $(ZLIB_LDFLAGS)
 BDB_LIBS       = @BDB_LIBS@ $(ZLIB_LIBS)
 
+##  SQLite support.  Additional flags and libraries used when compiling or
+##  linking code that contains SQLite support.
+
+SQLITE3_CPPFLAGS = @SQLITE3_CPPFLAGS@
+SQLITE3_LDFLAGS  = @SQLITE3_LDFLAGS@
+SQLITE3_LIBS     = @SQLITE3_LIBS@
+
 ##  INN libraries.  Nearly all INN programs are linked with libinn, and any
 ##  INN program that reads from or writes to article storage or overview is
 ##  linked against libstorage.	STORAGE_LIBS is for external libraries

Modified: configure.ac
===================================================================
--- configure.ac	2020-12-25 09:51:06 UTC (rev 10468)
+++ configure.ac	2020-12-25 09:57:59 UTC (rev 10469)
@@ -65,6 +65,7 @@
 m4_include([m4/snprintf.m4])
 m4_include([m4/socket-unix.m4])
 m4_include([m4/socket.m4])
+m4_include([m4/sqlite3.m4])
 m4_include([m4/syslog.m4])
 m4_include([m4/users.m4])
 m4_include([m4/vamacros.m4])
@@ -432,6 +433,7 @@
 INN_LIB_KRB5_OPTIONAL
 INN_LIB_OPENSSL_OPTIONAL
 INN_LIB_SASL_OPTIONAL
+INN_LIB_SQLITE3_OPTIONAL
 INN_LIB_ZLIB_OPTIONAL
 
 dnl If Kerberos is found, define KRB5_AUTH to auth_krb5 so as to build
@@ -516,9 +518,11 @@
 AC_TYPE_INT8_T
 AC_TYPE_INT16_T
 AC_TYPE_INT32_T
+AC_TYPE_INT64_T
 AC_TYPE_UINT8_T
 AC_TYPE_UINT16_T
 AC_TYPE_UINT32_T
+AC_TYPE_UINT64_T
 AC_TYPE_LONG_LONG_INT
 AC_TYPE_OFF_T
 AC_TYPE_PID_T

Modified: doc/FAQ
===================================================================
--- doc/FAQ	2020-12-25 09:51:06 UTC (rev 10468)
+++ doc/FAQ	2020-12-25 09:57:59 UTC (rev 10469)
@@ -452,13 +452,19 @@
 clients.
 
 Any INN server that supports readers must therefore have an overview
-method configured.  There are three different methods to choose from:
-tradindexed, which is the slowest but the best tested and most reliable
-and the method with the best recovery tools; buffindexed, which is fast at
-writing because it uses preconfigured large buffers like CNFS, but which
-is harder to recover; and the experimental ovdb overview method, which
-stores overview information in a BerkeleyDB database.
+method configured.  There are four different methods to choose from:
 
+  - buffindexed, which is fast at writing because it uses preconfigured
+    large buffers like CNFS, but which is hard to recover;
+  - ovdb, which stores overview information in a Berkeley DB database
+    and supports compression;
+  - ovsqlite, implemented in INN 2.7.0, which stores overview information
+    in an SQLite database and supports compression, but still not as
+    widely tested as the other overview mechanisms (all introduced with
+    INN 2.3.0);
+  - tradindexed, which is the slowest but the best tested, the most
+    reliable and the method with the best recovery tools.
+
 ------------------------------
 
 Subject: 2.5. What are deferrals (NNTP code 431)?

Index: trunk/doc/man
===================================================================
--- doc/man	2020-12-25 09:51:06 UTC (rev 10468)
+++ doc/man	2020-12-25 09:57:59 UTC (rev 10469)

Property changes on: trunk/doc/man
___________________________________________________________________
Modified: svn:ignore
## -69,6 +69,8 ##
 ovdb_server.8
 ovdb_stat.8
 overchan.8
+ovsqlite-server.8
+ovsqlite.5
 passwd.nntp.5
 perl-nocem.8
 pgpverify.1
Modified: doc/man/Makefile
===================================================================
--- doc/man/Makefile	2020-12-25 09:51:06 UTC (rev 10468)
+++ doc/man/Makefile	2020-12-25 09:57:59 UTC (rev 10469)
@@ -15,10 +15,11 @@
 	INN__Utils__Shlock.3pm
 
 SEC5	= active.5 active.times.5 buffindexed.conf.5 control.ctl.5 \
-	cycbuff.conf.5 distrib.pats.5 distributions.5 expire.ctl.5 history.5 incoming.conf.5 \
+	cycbuff.conf.5 distrib.pats.5 distributions.5 expire.ctl.5 \
+	history.5 incoming.conf.5 \
 	inn.conf.5 innfeed.conf.5 innwatch.ctl.5 moderators.5 motd.news.5 \
 	newsfeeds.5 newsgroups.5 newslog.5 nnrpd.track.5 nntpsend.ctl.5 ovdb.5 \
-	passwd.nntp.5 inn-radius.conf.5 readers.conf.5 \
+	ovsqlite.5 passwd.nntp.5 inn-radius.conf.5 readers.conf.5 \
 	storage.conf.5 subscriptions.5
 
 SEC8	= actsync.8 archive.8 batcher.8 buffchan.8 ckpasswd.8 \
@@ -29,7 +30,8 @@
 	innupgrade.8 innwatch.8 innxbatch.8 innxmit.8 mailpost.8 makedbz.8 \
 	makehistory.8 mod-active.8 news.daily.8 news2mail.8 ninpaths.8 \
 	nnrpd.8 nntpsend.8 ovdb_init.8 ovdb_monitor.8 ovdb_server.8 \
-	ovdb_stat.8 overchan.8 perl-nocem.8 procbatch.8 prunehistory.8 radius.8 \
+	ovdb_stat.8 overchan.8 ovsqlite-server.8 perl-nocem.8 procbatch.8 \
+	prunehistory.8 radius.8 \
 	rc.news.8 scanlogs.8 scanspool.8 send-nntp.8 send-uucp.8 sendinpaths.8 \
 	tally.control.8 tdx-util.8 tinyleaf.8 writelog.8
 

Modified: doc/pod/Makefile
===================================================================
--- doc/pod/Makefile	2020-12-25 09:51:06 UTC (rev 10468)
+++ doc/pod/Makefile	2020-12-25 09:57:59 UTC (rev 10469)
@@ -24,7 +24,7 @@
 	../man/distributions.5 ../man/expire.ctl.5 ../man/incoming.conf.5 \
 	../man/inn.conf.5 ../man/innfeed.conf.5 ../man/moderators.5 \
 	../man/motd.news.5 ../man/newsfeeds.5 ../man/newsgroups.5 \
-	../man/newslog.5 ../man/nntpsend.ctl.5 ../man/ovdb.5 \
+	../man/newslog.5 ../man/nntpsend.ctl.5 ../man/ovdb.5 ../man/ovsqlite.5 \
 	../man/passwd.nntp.5 ../man/inn-radius.conf.5 ../man/readers.conf.5 \
 	../man/storage.conf.5 ../man/subscriptions.5
 
@@ -31,7 +31,8 @@
 MAN8	= ../man/actsync.8 ../man/archive.8 ../man/auth_krb5.8 \
 	../man/batcher.8 ../man/buffchan.8 \
 	../man/ckpasswd.8 ../man/cnfsheadconf.8 ../man/cnfsstat.8 \
-	../man/controlchan.8 ../man/ctlinnd.8 ../man/cvtbatch.8 ../man/docheckgroups.8 \
+	../man/controlchan.8 ../man/ctlinnd.8 ../man/cvtbatch.8 \
+	../man/docheckgroups.8 \
 	../man/domain.8 ../man/expire.8 ../man/expireover.8 \
 	../man/expirerm.8 ../man/ident.8 \
 	../man/innbind.8 ../man/inncheck.8 ../man/innd.8 ../man/inndf.8 \
@@ -41,8 +42,8 @@
 	../man/news.daily.8 ../man/news2mail.8 ../man/ninpaths.8 \
 	../man/nnrpd.8 ../man/nntpsend.8 \
 	../man/ovdb_init.8 ../man/ovdb_monitor.8 ../man/ovdb_server.8 \
-	../man/ovdb_stat.8 ../man/overchan.8 \
-        ../man/procbatch.8 ../man/prunehistory.8 ../man/radius.8 \
+	../man/ovdb_stat.8 ../man/overchan.8 ../man/ovsqlite-server.8 \
+	../man/procbatch.8 ../man/prunehistory.8 ../man/radius.8 \
 	../man/rc.news.8 ../man/scanlogs.8 ../man/scanspool.8 \
 	../man/sendinpaths.8 \
 	../man/tally.control.8 ../man/tdx-util.8 \
@@ -105,6 +106,7 @@
 ../man/newslog.5:	newslog.pod		; $(POD2MAN) -s 5 $? > $@
 ../man/nntpsend.ctl.5:	nntpsend.ctl.pod	; $(POD2MAN) -s 5 $? > $@
 ../man/ovdb.5:		ovdb.pod		; $(POD2MAN) -s 5 $? > $@
+../man/ovsqlite.5:	ovsqlite.pod		; $(POD2MAN) -s 5 $? > $@
 ../man/passwd.nntp.5:	passwd.nntp.pod		; $(POD2MAN) -s 5 $? > $@
 ../man/inn-radius.conf.5: inn-radius.conf.pod	; $(POD2MAN) -s 5 $? > $@
 ../man/readers.conf.5:	readers.conf.pod	; $(POD2MAN) -s 5 $? > $@
@@ -149,6 +151,7 @@
 ../man/ovdb_server.8:	ovdb_server.pod		; $(POD2MAN) -s 8 $? > $@
 ../man/ovdb_stat.8:	ovdb_stat.pod		; $(POD2MAN) -s 8 $? > $@
 ../man/overchan.8:	overchan.pod		; $(POD2MAN) -s 8 $? > $@
+../man/ovsqlite-server.8: ovsqlite-server.pod	; $(POD2MAN) -s 8 $? > $@
 ../man/procbatch.8:	procbatch.pod		; $(POD2MAN) -s 8 $? > $@
 ../man/prunehistory.8:	prunehistory.pod	; $(POD2MAN) -s 8 $? > $@
 ../man/radius.8:	radius.pod		; $(POD2MAN) -s 8 $? > $@

Modified: doc/pod/checklist.pod
===================================================================
--- doc/pod/checklist.pod	2020-12-25 09:51:06 UTC (rev 10468)
+++ doc/pod/checklist.pod	2020-12-25 09:57:59 UTC (rev 10469)
@@ -92,7 +92,7 @@
 
 You probably want B<--with-perl>.  If you're not using NetBSD with
 cycbuffs or OpenBSD, perhaps B<--with-tagged-hash>.  You might want to
-compile in TLS/SSL and S<Berkeley DB>, if your system supports them.  You
+compile in TLS/SSL and SQLite, if your system supports them.  You
 will need to have the relevant external libraries to compile (depending
 on whether you use OpenSSL for TLS/SSL access to your news server, GnuPG
 to verify the authenticity of Usenet control messages, Perl, Python, etc.).

Modified: doc/pod/inn.conf.pod
===================================================================
--- doc/pod/inn.conf.pod	2020-12-25 09:51:06 UTC (rev 10468)
+++ doc/pod/inn.conf.pod	2020-12-25 09:57:59 UTC (rev 10469)
@@ -590,8 +590,9 @@
 =item I<ovmethod>
 
 Which overview storage method to use.  Currently supported values are
-C<tradindexed>, C<buffindexed>, and C<ovdb>.  There is no default value;
-this parameter must be set if I<enableoverview> is true (the default).
+C<buffindexed>, C<ovdb>, C<ovsqlite> and C<tradindexed>.  There is no
+default value; this parameter must be set if I<enableoverview> is true
+(the default).
 
 =over 4
 
@@ -601,15 +602,19 @@
 preconfigured files defined in F<buffindexed.conf>.  C<buffindexed> never
 consumes additional disk space beyond that allocated to these buffers.
 
+=item C<ovdb>
+
+Stores data into a S<Berkeley DB> database.  See the ovdb(5) man page.
+
+=item C<ovsqlite>
+
+Stores data into an SQLite database.  See the ovsqlite(5) man page.
+
 =item C<tradindexed>
 
 Uses two files per newsgroup, one containing the overview data and one
 containing the index.  Fast for readers, but slow to write to.
 
-=item C<ovdb>
-
-Stores data into a S<Berkeley DB> database.  See the ovdb(5) man page.
-
 =back
 
 =item I<storeonxref>

Modified: doc/pod/install.pod
===================================================================
--- doc/pod/install.pod	2020-12-25 09:51:06 UTC (rev 10468)
+++ doc/pod/install.pod	2020-12-25 09:57:59 UTC (rev 10469)
@@ -148,6 +148,7 @@
     --with-perl         Perl 5.004_03 or higher, 5.8.0+ recommended
     --with-python       Python 2.3.0 or higher, 2.5.0+ recommended (in the 2.x series); Python 3.3.0 or higher (in the 3.x series)
     --with-bdb          Berkeley DB 4.4 or higher, 4.7+ recommended
+    --with-sqlite3      SQLite 3.8.2 or higher, 3.20.0+ recommended
     --with-zlib         zlib 1.x or higher
     --with-openssl      OpenSSL 0.9.6 or higher
     --with-sasl         Cyrus SASL 2.x or higher
@@ -423,6 +424,22 @@
 built with S<Berkeley DB> support unless the B<--without-bdb> flag is
 explicitly passed to configure.
 
+=item B<--with-sqlite3>=PATH
+
+Enables support for SQLite (3.8.2 or higher), which means that it
+will then be possible to use the ovsqlite overview method if you wish.
+Enabling this configure option doesn't mean you'll be required to use
+ovsqlite, but it does require that SQLite be installed on your system
+(including the header files, not just the runtime libraries).  If a
+path is given, it sets the installed directory of SQLite.  In case
+non-standard paths to the SQLite library is used, one or both of the
+options B<--with-sqlite3-include> and B<--with-sqlite3-lib> can be given
+to configure with a path.
+
+If the SQLite library is found at configure time, INN will be built
+with SQLite support unless the B<--without-sqlite3> flag is explicitly
+passed to configure.
+
 =item B<--with-zlib>=PATH
 
 Enables support for compression for news reading, which means a
@@ -429,7 +446,8 @@
 compression layer can be negotiated between your server and newsreaders
 supporting that NNTP extension.
 
-Also enables support for compression with the ovdb storage method.
+Also enables support for compression with the ovdb and ovsqlite overview
+storage methods.
 
 This option requires that zlib be installed on your system (including the
 header files, not just the runtime libraries).  If a path is given, it
@@ -666,11 +684,6 @@
 
 =over 4
 
-=item tradindexed
-
-It is very fast for readers, but it has to update two files for each
-incoming article and can be quite slow to write.
-
 =item buffindexed
 
 It can keep up with a large feed more easily, since it uses large buffers
@@ -682,10 +695,24 @@
 
 =item ovdb
 
-It stores overview data in a S<Berkeley DB> database; it's fast and very robust,
-but may require more disk space.  See the ovdb(5) man page for more
-information on it.
+It stores overview data in a S<Berkeley DB> database; it is fast and very
+robust, but may require more disk space, unless compression is enabled.
+See the ovdb(5) man page for more information on it.
 
+=item ovsqlite
+
+It stores overview data in an SQLite database, known for its long-term
+stability and compatibility.  It is robust and faster at reading ranges
+of overview data, but somewhat slower at writing.  It may also require
+more disk space, unless compression is enabled.  See the ovsqlite(5)
+man page for more information on it.
+
+=item tradindexed
+
+It is very fast for readers, but it has to update two files for each
+incoming article and can be quite slow to write.  Robust and well-tested,
+with the best recovery tools.
+
 =back
 
 =head1 Configuring INN

Modified: doc/pod/makehistory.pod
===================================================================
--- doc/pod/makehistory.pod	2020-12-25 09:51:06 UTC (rev 10468)
+++ doc/pod/makehistory.pod	2020-12-25 09:57:59 UTC (rev 10469)
@@ -24,7 +24,9 @@
 manager, and write a history line for every article.  To also generate
 overview information, use the B<-O> flag.
 
-WARNING:  If you're trying to rebuild the overview database, be sure to
+=head1 OVERVIEW REBUILD
+
+I<WARNING>:  If you're trying to rebuild the overview database, be sure to
 stop innd(8) and delete or zero out the existing database before you start
 for the best results.  An overview rebuild should not be done while the
 server is running.  Unless the existing overview is deleted, you may end
@@ -37,6 +39,55 @@
 rest of the server by running B<ovdb_init>; see ovdb_init(8) for more
 details.
 
+Similarly, if I<ovmethod> in F<inn.conf> is C<ovsqlite>, you must
+have the B<ovsqlite-server> process running while rebuilding overview.
+See ovsqlite-server(8) for more details and how to start it by hand.
+
+Rebuilding overview data is as straight-forward as:
+
+=over 4
+
+=item 1.
+
+Setting the new overview storage method in the I<ovmethod> parameter
+in F<inn.conf>.
+
+=item 2.
+
+Checking that its configuration file is correctly installed in
+I<pathetc> and fits your needs (F<buffindexed.conf>, F<ovdb.conf> or
+F<ovsqlite.conf>).  Note that the tradindexed overview storage method
+does not have a configuration file.
+
+=item 3.
+
+Making sure that INN is stopped.
+
+=item 4.
+
+Making sure that the directory specified by the I<pathoverview> parameter
+in F<inn.conf> exists and is empty.  Otherwise, rename the current one
+(to backup existing overview data) and re-create I<pathoverview> as
+the news user.
+
+=item 5.
+
+Starting B<ovdb_init> or B<ovsqlite-server> as the news user if the
+new overview storage method is respectively ovdb or ovsqlite.
+
+=item 6.
+
+Running C<makehistory -O -x -F> and waiting for the command to finish.
+
+=item 7.
+
+Starting INN and checking the logs to make sure everything is fine.
+You will normally notice that the F<active> file is renumbered
+(B<rc.news> takes care of that when run after an overview rebuild;
+otherwise, manually run C<ctlinnd renumber ''>).
+
+=back
+
 =head1 OPTIONS
 
 =over 4
@@ -201,6 +252,6 @@
 =head1 SEE ALSO
 
 active(5), ctlinnd(8), dbz(3), history(5), inn.conf(5), innd(8),
-makedbz(8), ovdb_init(8), overchan(8).
+makedbz(8), ovdb_init(8), overchan(8), ovsqlite-server(8).
 
 =cut

Modified: doc/pod/news.pod
===================================================================
--- doc/pod/news.pod	2020-12-25 09:51:06 UTC (rev 10468)
+++ doc/pod/news.pod	2020-12-25 09:57:59 UTC (rev 10469)
@@ -1,3 +1,22 @@
+=head1 Changes in 2.7.0
+
+=over 2
+
+=item *
+
+Bo Lindbergh has implemented a new overview storage method based
+on SQLite, known for its long-term stability and compatibility.
+Robust and faster at reading ranges of overview data, but somewhat
+slower at writing, this new SQLite-based method is a perfect choice to
+store overview data.
+
+To select it as your overview method, set the I<ovmethod> parameter in
+F<inn.conf> to C<ovsqlite>.  Details about ovsqlite and how to switch to
+that new modern overview storage method can be found in the ovsqlite(5)
+and makehistory(8) man pages.
+
+=back
+
 =head1 Changes in 2.6.4
 
 =over 2

Added: doc/pod/ovsqlite-server.pod
===================================================================
--- doc/pod/ovsqlite-server.pod	                        (rev 0)
+++ doc/pod/ovsqlite-server.pod	2020-12-25 09:57:59 UTC (rev 10469)
@@ -0,0 +1,75 @@
+=head1 NAME
+
+ovsqlite-server - Sole owner of the ovsqlite database
+
+=head1 SYNOPSIS
+
+B<ovsqlite-server> [B<-d>]
+
+=head1 DESCRIPTION
+
+The B<ovsqlite-server> daemon is the only program that opens the overview
+SQLite database.  It accepts connections from the other parts of INN that
+want to operate on overview data (B<innd>, B<nnrpd>, B<expireover>,
+B<makehistory>).
+
+This daemon must therefore be started before any other process can
+access the overview database.  B<ovsqlite-server> is normally invoked
+automatically by B<rc.news> when starting the news system.
+
+
+=head1 OPTIONS
+
+=over 4
+
+=item B<-d>
+
+B<ovsqlite-server> normally puts itself into the background, points
+its standard output and error to log files, and disassociates itself
+from the terminal.  Using B<-d> prevents all of this, resulting in log
+messages being written to the standard error output; this is generally
+useful only for debugging.
+
+=back
+
+=head1 FILES
+
+=over 4
+
+=item I<pathetc>/ovsqlite.conf
+
+The configuration file.  See ovsqlite(5).
+
+=item I<pathoverview>/ovsqlite.db
+
+The SQLite database file.
+
+=item I<pathrun>/ovsqlite.pid
+
+Stores the PID of the server process while it's running.
+
+=item I<pathrun>/ovsqlite.sock
+
+When Unix-domain sockets are available, the server binds its listening
+socket to this path.
+
+=item I<pathrun>/ovsqlite.port
+
+When Unix-domain sockets I<aren't> available, the server binds its
+listening socket to a dynamic TCP port on the IPv4 loopback interface and
+stores the port number in this file.
+
+=back
+
+=head1 HISTORY
+
+Initial implementation of ovsqlite written by Bo Lindbergh
+<2bfjdsla52kztwejndzdstsxl9athp at gmail.com> for InterNetNews.
+
+$Id$
+
+=head1 SEE ALSO
+
+ovsqlite(5), rc.news(8).
+
+=cut


Property changes on: trunk/doc/pod/ovsqlite-server.pod
___________________________________________________________________
Added: svn:eol-style
## -0,0 +1 ##
+native
\ No newline at end of property
Added: svn:keywords
## -0,0 +1 ##
+Author Date Id Revision
\ No newline at end of property
Added: doc/pod/ovsqlite.pod
===================================================================
--- doc/pod/ovsqlite.pod	                        (rev 0)
+++ doc/pod/ovsqlite.pod	2020-12-25 09:57:59 UTC (rev 10469)
@@ -0,0 +1,107 @@
+=head1 NAME
+
+ovsqlite - SQLite-based overview storage method for INN
+
+=head1 DESCRIPTION
+
+This method uses SQLite to store overview data.  It requires version
+3.8.2 or later of the SQLite library (3.20.0+ recommended).
+
+SQLite source, documentation, etc. are available at
+L<https://www.sqlite.org/>.  Ones of the stated goals of the SQLite
+file format are long-term stability and compatibility, which make that
+storage method a perfect choice to store overview data.
+
+Only one protocol version of the ovsqlite storage method currently
+exists, implemented since S<INN 2.7.0>.
+
+=head1 INSTALLATION
+
+The configure script will automatically enable ovsqlite support if it finds
+the SQLite library.  If the library isn't installed in a standard
+location, you may have to specify the B<--with-sqlite3> option to help
+configure find it.  For complicated cases, you can use separate
+B<--with-sqlite3-include> and B<--with-sqlite3-lib> options.  Finally, if
+you I<don't> want ovsqlite support even when your system has the SQLite
+library, you can use the B<--without-sqlite3> option.
+
+If you have a recent Linux installation, SQLite is most likely already
+installed.  You may have to install a separate package with a name similar
+to C<libsqlite3-dev> to get the required header files.
+
+=head1 CONFIGURATION
+
+To select ovsqlite as your overview method, set the I<ovmethod> parameter
+in F<inn.conf> to C<ovsqlite>.  The database file will be created in
+the directory specified by the I<pathoverview> parameter in F<inn.conf>.
+Restart INN to take the change into account (after having rebuilt your
+existing overview with B<makehistory>, if needed).
+
+These additional parameters are read from the F<ovsqlite.conf>
+configuration file:
+
+=over 4
+
+=item I<cachesize>
+
+The SQLite in-memory page cache size in kilobytes.  The default value is
+left up to the SQLite library and seems to be stable at S<2000 KB>.
+
+=item I<compress>
+
+If INN was built with zlib support and this parameter is true, ovsqlite
+will compress overview records whenever this saves space.  This parameter
+is consulted only when creating a new database.  Enabling compression
+saves about S<70 %> of disk space on typical overview data. The default
+value is false.
+
+=item I<pagesize>
+
+The SQLite database page size in bytes.  Must be a power of 2, minimum 512,
+maximum 65536.  Appropriate values include the virtual memory page size and
+the filesystem allocation block size.  This parameter is consulted only
+when creating a new database.  The default value is left up to the SQLite
+library and varies between versions.
+
+=item I<transrowlimit>
+
+The maximum number of article rows that can be inserted or deleted in a
+single SQL transaction.  The default value is 10000 articles.
+
+=item I<transtimelimit>
+
+The maximum SQL transaction lifetime in seconds.  The default value is
+10 seconds.
+
+=back
+
+A transaction occurs every I<transrowlimit> articles or I<transtimelimit>
+seconds, whichever is smaller.  You are encouraged to keep the default
+value for row limits and, instead, adjust the time limit according to
+how many articles your news server usually accepts per second during
+normal operation (you can find statistics about incoming articles in
+your daily Usenet reports).  Inserting or deleting a database row within
+a transaction is very fast whereas committing a transaction is slow,
+especially on rotating storage.  Setting transaction limits too low
+leads to poor performance.  When rebuilding overview data, it may be
+worth temporarily raising these values, though.
+
+=head1 RUNNING
+
+All overview database access goes through the B<sqlite-server> daemon.  For
+ordinary operation, B<rc.news> will start and stop it automatically.  If
+you want to touch the overview database while B<innd> isn't running, you'll
+have to start B<sqlite-server> manually first.  See sqlite-server(8).
+
+=head1 HISTORY
+
+Initial implementation of ovsqlite written by Bo Lindbergh
+<2bfjdsla52kztwejndzdstsxl9athp at gmail.com> for InterNetNews.
+
+$Id$
+
+=head1 SEE ALSO
+
+inn.conf(5), makehistory(8), rc.news(8), sqlite-server(8).
+
+=cut


Property changes on: trunk/doc/pod/ovsqlite.pod
___________________________________________________________________
Added: svn:eol-style
## -0,0 +1 ##
+native
\ No newline at end of property
Added: svn:keywords
## -0,0 +1 ##
+Author Date Id Revision
\ No newline at end of property
Modified: doc/pod/rc.news.pod
===================================================================
--- doc/pod/rc.news.pod	2020-12-25 09:51:06 UTC (rev 10468)
+++ doc/pod/rc.news.pod	2020-12-25 09:57:59 UTC (rev 10469)
@@ -41,6 +41,11 @@
 
 =item *
 
+If I<ovmethod> is set to C<ovsqlite> in F<inn.conf>:  B<ovsqlite-server>
+is started and stopped.
+
+=item *
+
 If F<rc.news.local> exists in I<pathbin>:  B<rc.news.local> is run with
 argument C<start> or C<stop> (to perform site-specific startup or shutdown
 tasks).
@@ -103,6 +108,7 @@
 
 =head1 SEE ALSO
 
-ctlinnd(8), cnfsstat(8), expirerm(8), inn.conf(5), innwatch(8), ovdb(5).
+ctlinnd(8), cnfsstat(8), expirerm(8), inn.conf(5), innwatch(8), ovdb(5),
+ovsqlite(5).
 
 =cut

Added: m4/sqlite3.m4
===================================================================
--- m4/sqlite3.m4	                        (rev 0)
+++ m4/sqlite3.m4	2020-12-25 09:57:59 UTC (rev 10469)
@@ -0,0 +1,87 @@
+dnl Find the compiler and linker flags for SQLite v3.
+dnl $Id$
+dnl
+dnl Finds the compiler and linker flags for linking with the SQLite library.
+dnl Provides the --with-sqlite3, --with-sqlite3-lib, and
+dnl --with-sqlite3-include configure options to specify non-standard paths to
+dnl the SQLite v3 libraries or header files.
+dnl
+dnl Provides the macros INN_LIB_SQLITE3 and INN_LIB_SQLITE3_OPTIONAL and sets
+dnl the substitution variables SQLITE3_CPPFLAGS, SQLITE3_LDFLAGS, and
+dnl SQLITE3_LIBS.  Also provides INN_LIB_SQLITE3_SWITCH to set CPPFLAGS,
+dnl LDFLAGS, and LIBS to include the SQLite libraries, saving the current
+dnl values first, and INN_LIB_SQLITE3_RESTORE to restore those settings to
+dnl before the last INN_LIB_SQLITE3_SWITCH.  Defines HAVE_SQLITE3 and sets
+dnl inn_use_SQLITE3 to true if SQLite 3 is found.  If it isn't found, the
+dnl substitution variables will be empty.
+dnl
+dnl Depends on the lib-helper.m4 framework.
+dnl
+dnl The canonical version of this file is maintained in the rra-c-util
+dnl package, available at <https://www.eyrie.org/~eagle/software/rra-c-util/>.
+dnl
+dnl Written by Russ Allbery <eagle at eyrie.org>
+dnl Copyright 2020 Russ Allbery <eagle at eyrie.org>
+dnl Copyright 2014
+dnl     The Board of Trustees of the Leland Stanford Junior University
+dnl
+dnl This file is free software; the authors give unlimited permission to copy
+dnl and/or distribute it, with or without modifications, as long as this
+dnl notice is preserved.
+dnl
+dnl SPDX-License-Identifier: FSFULLR
+
+dnl Save the current CPPFLAGS, LDFLAGS, and LIBS settings and switch to
+dnl versions that include the libevent flags.  Used as a wrapper, with
+dnl INN_LIB_SQLITE3_RESTORE, around tests.
+AC_DEFUN([INN_LIB_SQLITE3_SWITCH], [INN_LIB_HELPER_SWITCH([SQLITE3])])
+
+dnl Restore CPPFLAGS, LDFLAGS, and LIBS to their previous values before
+dnl INN_LIB_SQLITE3_SWITCH was called.
+AC_DEFUN([INN_LIB_SQLITE3_RESTORE], [INN_LIB_HELPER_RESTORE([SQLITE3])])
+
+dnl Ensures SQLite v3 meets our minimum version requirement.
+AC_DEFUN([_INN_LIB_SQLITE3_SOURCE], [[
+#include <sqlite3.h>
+
+int main(void) {
+    return sqlite3_libversion_number() < 3008002;
+}
+]])
+
+dnl Checks if SQLite v3 is present.  The single argument, if "true", says to
+dnl fail if the SQLite library could not be found.
+AC_DEFUN([_INN_LIB_SQLITE3_INTERNAL],
+[AC_CACHE_CHECK([for a sufficiently recent SQLite],
+    [inn_cv_have_sqlite3],
+    [INN_LIB_HELPER_PATHS([SQLITE3])
+     INN_LIB_SQLITE3_SWITCH
+     LIBS="-lsqlite3 $LIBS"
+     AC_RUN_IFELSE([AC_LANG_SOURCE([_INN_LIB_SQLITE3_SOURCE])],
+        [inn_cv_have_sqlite3=yes],
+        [inn_cv_have_sqlite3=no])
+     INN_LIB_SQLITE3_RESTORE])
+ AS_IF([test x"$inn_cv_have_sqlite3" = xyes],
+    [SQLITE3_LIBS="-lsqlite3"],
+    [AS_IF([test x"$1" = xtrue],
+        [AC_MSG_ERROR([cannot find usable SQLite v3 library])])])])
+
+dnl The main macro for packages with mandatory SQLite v3 support.
+AC_DEFUN([INN_LIB_SQLITE3],
+[INN_LIB_HELPER_VAR_INIT([SQLITE3])
+ INN_LIB_HELPER_WITH([sqlite3], [SQLite v3], [SQLITE3])
+ _INN_LIB_SQLITE3_INTERNAL([true])
+ inn_use_SQLITE3=true
+ AC_DEFINE([HAVE_SQLITE3], 1, [Define if SQLite v3 is available.])])
+
+dnl The main macro for packages with optional SQLite v3 support.
+AC_DEFUN([INN_LIB_SQLITE3_OPTIONAL],
+[INN_LIB_HELPER_VAR_INIT([SQLITE3])
+ INN_LIB_HELPER_WITH_OPTIONAL([sqlite3], [SQLite v3], [SQLITE3])
+ AS_IF([test x"$inn_use_SQLITE3" != xfalse],
+    [AS_IF([test x"$inn_use_SQLITE3" = xtrue],
+        [_INN_LIB_SQLITE3_INTERNAL([true])],
+        [_INN_LIB_SQLITE3_INTERNAL([false])])])
+ AS_IF([test x"$SQLITE3_LIBS" != x],
+    [inn_use_SQLITE3=true
+     AC_DEFINE([HAVE_SQLITE3], 1, [Define if SQLite v3 is available.])])])

Added: samples/ovsqlite.conf
===================================================================
--- samples/ovsqlite.conf	                        (rev 0)
+++ samples/ovsqlite.conf	2020-12-25 09:57:59 UTC (rev 10469)
@@ -0,0 +1,46 @@
+# The directory that overview will be stored in is set in inn.conf with
+# the 'pathoverview' option.  Other parameters for tuning ovsqlite are
+# in this file.
+
+# Compression: if INN was built with zlib support and this parameter
+# is true, ovsqlite will compress overview records whenever this saves
+# space.  This parameter is consulted only when creating a new database.
+# Enabling compression saves about S<70 %> of disk space on typical
+# overview data.
+# The default value is false.
+#compress:              false
+
+# The SQLite database page size in bytes.
+# Must be a power of 2, minimum 512, maximum 65536.
+# Appropriate values include the virtual memory page size and the
+# filesystem allocation block size.
+# This parameter is consulted only when creating a new database.
+# The default value is left up to the SQLite library and varies
+# between versions.
+#pagesize:              4096
+
+# The SQLite in-memory page cache size in kilobytes.
+# The default value is left up to the SQLite library and seems to be
+# stable at 2000 KB.
+#cachesize:             2000
+
+# The maximum number of article rows that can be inserted or deleted
+# in a single SQL transaction.
+# The default value is 10000 articles.
+#transrowlimit:         10000
+
+# The maximum SQL transaction lifetime in seconds.
+# The default value is 10 seconds.
+#transtimelimit:        10.0
+
+# A transaction occurs every transrowlimit articles or transtimelimit
+# seconds, whichever is smaller.  You are encouraged to keep the default
+# value for row limits and, instead, adjust the time limit according to
+# how many articles your news server usually accepts per second during
+# normal operation (you can find statistics about incoming articles in
+# your daily Usenet reports).
+# Inserting or deleting a database row within a transaction is very fast
+# whereas committing a transaction is slow, especially on rotating
+# storage.  Setting transaction limits too low leads to poor
+# performance.  When rebuilding overview data, it may be worth
+# temporarily raising these values, though.


Property changes on: trunk/samples/ovsqlite.conf
___________________________________________________________________
Added: svn:eol-style
## -0,0 +1 ##
+native
\ No newline at end of property
Added: svn:keywords
## -0,0 +1 ##
+Author Date Id Revision
\ No newline at end of property
Modified: scripts/rc.news.in
===================================================================
--- scripts/rc.news.in	2020-12-25 09:51:06 UTC (rev 10468)
+++ scripts/rc.news.in	2020-12-25 09:57:59 UTC (rev 10469)
@@ -46,23 +46,38 @@
 	printf "\n"
     fi
 
-    # Turn off ovdb support procs, and close the DB environment
-    if [ "$OVMETHOD" = "ovdb" -a -f ${PATHRUN}/ovdb_server.pid ]; then
-	pid=`cat ${PATHRUN}/ovdb_server.pid 2>/dev/null`
-	if [ "$pid" != "" ]; then
-	    printf "Stopping ovdb_server: "
-	    kill $pid
-	    waitforpid $pid
+    case "$OVMETHOD" in
+    ovdb)
+	# Turn off ovdb support procs, and close the DB environment
+	if [ -f ${PATHRUN}/ovdb_server.pid ]; then
+	    pid=`cat ${PATHRUN}/ovdb_server.pid 2>/dev/null`
+	    if [ "$pid" != "" ]; then
+		printf "Stopping ovdb_server: "
+		kill $pid
+		waitforpid $pid
+	    fi
 	fi
-    fi
-    if [ "$OVMETHOD" = "ovdb" -a -f ${PATHRUN}/ovdb_monitor.pid ]; then
-	pid=`cat ${PATHRUN}/ovdb_monitor.pid 2>/dev/null`
-	if [ "$pid" != "" ]; then
-	    printf "Stopping ovdb_monitor: "
-	    kill $pid
-	    waitforpid $pid
+	if [ -f ${PATHRUN}/ovdb_monitor.pid ]; then
+	    pid=`cat ${PATHRUN}/ovdb_monitor.pid 2>/dev/null`
+	    if [ "$pid" != "" ]; then
+		printf "Stopping ovdb_monitor: "
+		kill $pid
+		waitforpid $pid
+	    fi
 	fi
-    fi
+	;;
+    ovsqlite)
+        # Turn off ovsqlite support proc, and close the DB environment
+	if [ -f "${PATHRUN}"/ovsqlite.pid ]; then
+	    pid=`cat "${PATHRUN}"/ovsqlite.pid 2>/dev/null`
+	    if [ "$pid" != "" ]; then
+		printf "Stopping ovsqlite-server: "
+		kill $pid
+		waitforpid $pid
+	    fi
+	fi
+	;;
+    esac
 
     if [ -f ${PATHBIN}/rc.news.local ]; then
         ${PATHBIN}/rc.news.local stop
@@ -72,6 +87,7 @@
     # stick around after a fresh start.
     rm -f ${PATHRUN}/cnfsstat.pid $WATCHPID $SERVERPID
     rm -f ${PATHRUN}/ovdb_server.pid ${PATHRUN}/ovdb_monitor.pid
+    rm -f ${PATHRUN}/ovsqlite.pid
 
     exit 0
 ;;
@@ -146,8 +162,9 @@
 ( cd ${TEMPSOCKDIR} && rm -f ${TEMPSOCK} )
 rm -f ${NEWSCONTROL} ${NNTPCONNECT} ${SERVERPID} ${PATHRUN}/.rebuildoverview
 
-## Initialize ovdb.  Must be done before starting innd.
-if [ "$OVMETHOD" = "ovdb" ]; then
+case "$OVMETHOD" in
+ovdb)
+    # Initialize ovdb.  Must be done before starting innd.
     echo 'Starting ovdb.'
     ovdb_init || {
 	echo "Can't initialize ovdb (check news.err for OVDB messages)"
@@ -154,7 +171,14 @@
 	exit 1
     }
     sleep 1
-fi
+    ;;
+ovsqlite)
+    # Start ovsqlite-server.  Must be done before starting innd.
+    echo 'Starting ovsqlite-server.'
+    ovsqlite-server
+    sleep 1
+    ;;
+esac
 
 ##  Start the show.
 echo 'Starting innd.'

Index: trunk/site
===================================================================
--- site	2020-12-25 09:51:06 UTC (rev 10468)
+++ site	2020-12-25 09:57:59 UTC (rev 10469)

Property changes on: trunk/site
___________________________________________________________________
Modified: svn:ignore
## -41,6 +41,7 ##
 nntpsend.ctl
 nocem.ctl
 ovdb.conf
+ovsqlite.conf
 passwd.nntp
 readers.conf
 sasl.conf
Modified: site/Makefile
===================================================================
--- site/Makefile	2020-12-25 09:51:06 UTC (rev 10468)
+++ site/Makefile	2020-12-25 09:57:59 UTC (rev 10469)
@@ -48,6 +48,7 @@
 PATH_BUFFINDEXED	= ${PATHETC}/buffindexed.conf
 PATH_RADIUS_CONF	= ${PATHETC}/inn-radius.conf
 PATH_OVDB_CONF		= ${PATHETC}/ovdb.conf
+PATH_OVSQLITE_CONF	= ${PATHETC}/ovsqlite.conf
 PATH_SENDUUCP_CF	= ${PATHETC}/send-uucp.cf
 PATH_SUBSCRIPTIONS	= ${PATHETC}/subscriptions
 
@@ -70,7 +71,7 @@
 	nnrpd_auth.pl nnrpd_access.pl nocem.ctl \
         news2mail.cf readers.conf \
 	inn-radius.conf nnrpd_auth.py nnrpd_access.py nnrpd_dynamic.py \
-	ovdb.conf active.minimal \
+	ovdb.conf ovsqlite.conf active.minimal \
 	newsgroups.minimal send-uucp.cf subscriptions
 
 ALL		= $(REST)
@@ -96,8 +97,8 @@
 	$D$(PATHETC)/innshellvars.tcl.local \
 	$D$(PATHETC)/nocem.ctl \
 	$D$(PATH_NNRPAUTH) $D$(PATHETC)/news2mail.cf $D$(PATH_READERSCONF) \
-	$D$(PATH_RADIUS_CONF) $D$(PATH_NNRPYAUTH) $D$(PATH_NNRPYACCESS) $D$(PATH_NNRPYDYNAMIC) \
-	$D$(PATH_OVDB_CONF) \
+	$D$(PATH_RADIUS_CONF) $D$(PATH_NNRPYAUTH) $D$(PATH_NNRPYACCESS) \
+	$D$(PATH_NNRPYDYNAMIC) $D$(PATH_OVDB_CONF) $D$(PATH_OVSQLITE_CONF) \
 	$D$(PATH_SENDUUCP_CF) $D$(PATH_SUBSCRIPTIONS) $D$(PATH_NNRPACCESS)
 
 ALL_INSTALLED	= $(REST_INSTALLED)
@@ -212,6 +213,7 @@
 $D$(PATH_CYCBUFFCONFIG): cycbuff.conf	; $(COPY_RPUB) $? $@
 $D$(PATH_BUFFINDEXED): buffindexed.conf	; $(COPY_RPUB) $? $@
 $D$(PATH_OVDB_CONF): ovdb.conf		; $(COPY_RPUB) $? $@
+$D$(PATH_OVSQLITE_CONF): ovsqlite.conf	; $(COPY_RPUB) $? $@
 $D$(PATH_PERL_STARTUP_INND): startup_innd.pl ; $(COPY_RPUB) $? $@
 $D$(PATH_PERL_FILTER_INND): filter_innd.pl ; $(COPY_RPUB) $? $@
 $D$(PATH_PERL_FILTER_NNRPD): filter_nnrpd.pl ; $(COPY_RPUB) $? $@
@@ -272,6 +274,7 @@
 cycbuff.conf:	../samples/cycbuff.conf		; $(COPY) $? $@
 buffindexed.conf: ../samples/buffindexed.conf	; $(COPY) $? $@
 ovdb.conf:	../samples/ovdb.conf		; $(COPY) $? $@
+ovsqlite.conf:	../samples/ovsqlite.conf	; $(COPY) $? $@
 innwatch.ctl:	../samples/innwatch.ctl		; $(COPY) $? $@
 innfeed.conf:	../samples/innfeed.conf		; $(COPY) $? $@
 moderators:	../samples/moderators		; $(COPY) $? $@

Modified: storage/Makefile
===================================================================
--- storage/Makefile	2020-12-25 09:51:06 UTC (rev 10468)
+++ storage/Makefile	2020-12-25 09:57:59 UTC (rev 10469)
@@ -11,7 +11,7 @@
 LTVERSION     = 3:4:0
 
 top	      = ..
-CFLAGS	      = $(GCFLAGS) -I. $(BDB_CPPFLAGS)
+CFLAGS	      = $(GCFLAGS) -I. $(BDB_CPPFLAGS) $(SQLITE3_CPPFLAGS)
 
 SOURCES	      = expire.c interface.c methods.c ov.c overdata.c overview.c \
 		ovmethods.c $(METHOD_SOURCES)
@@ -179,7 +179,8 @@
   ../include/inn/defines.h ../include/inn/options.h \
   ../include/inn/history.h ../include/inn/ov.h ../include/inn/storage.h \
   ../include/inn/options.h ../include/inn/storage.h \
-  buffindexed/buffindexed.h ovdb/ovdb.h tradindexed/tradindexed.h
+  buffindexed/buffindexed.h ovdb/ovdb.h ovsqlite/ovsqlite.h \
+  tradindexed/tradindexed.h
 buffindexed/buffindexed.o: buffindexed/buffindexed.c ../include/config.h \
   ../include/inn/defines.h ../include/inn/system.h ../include/inn/macros.h \
   ../include/inn/portable-macros.h ../include/inn/portable-stdbool.h \
@@ -233,6 +234,31 @@
   ../include/portable/socket-unix.h ../include/inn/ov.h \
   ../include/inn/history.h ../include/inn/storage.h ovinterface.h \
   ovdb/ovdb.h ovdb/ovdb-private.h
+ovsqlite/ovsqlite-private.o: ovsqlite/ovsqlite-private.c \
+  ovsqlite/ovsqlite-private.h ../include/config.h ../include/inn/defines.h \
+  ../include/inn/system.h ../include/inn/macros.h \
+  ../include/inn/portable-macros.h ../include/inn/portable-stdbool.h \
+  ../include/inn/defines.h ../include/inn/options.h ../include/clibrary.h \
+  ../include/config.h ../include/inn/macros.h \
+  ../include/portable/stdbool.h ../include/portable/macros.h \
+  ../include/inn/buffer.h ../include/inn/xmalloc.h
+ovsqlite/ovsqlite.o: ovsqlite/ovsqlite.c ovsqlite/ovsqlite.h ../include/config.h \
+  ../include/inn/defines.h ../include/inn/system.h ../include/inn/macros.h \
+  ../include/inn/portable-macros.h ../include/inn/portable-stdbool.h \
+  ../include/inn/defines.h ../include/inn/options.h \
+  ../include/inn/storage.h ../include/inn/options.h ../include/inn/ov.h \
+  ../include/inn/history.h ../include/inn/storage.h \
+  ../include/inn/messages.h ovsqlite/ovsqlite-private.h \
+  ../include/clibrary.h ../include/config.h ../include/inn/macros.h \
+  ../include/portable/stdbool.h ../include/portable/macros.h \
+  ../include/inn/buffer.h ../include/portable/socket.h \
+  ../include/portable/getaddrinfo.h ../include/portable/getnameinfo.h \
+  ../include/portable/socket-unix.h ../include/conffile.h \
+  ../include/portable/macros.h ../include/inn/fdflag.h \
+  ../include/inn/portable-socket.h ../include/inn/innconf.h \
+  ../include/inn/libinn.h ../include/inn/concat.h ../include/inn/xmalloc.h \
+  ../include/inn/xwrite.h ../include/inn/newsuser.h ../include/inn/paths.h \
+  ovsqlite/../ovinterface.h
 timecaf/caf.o: timecaf/caf.c ../include/config.h ../include/inn/defines.h \
   ../include/inn/system.h ../include/inn/macros.h \
   ../include/inn/portable-macros.h ../include/inn/portable-stdbool.h \
@@ -337,6 +363,37 @@
   ../include/inn/concat.h ../include/inn/xmalloc.h ../include/inn/xwrite.h \
   methods.h interface.h ../include/inn/storage.h ../include/inn/options.h \
   trash/trash.h interface.h
+ovsqlite/ovsqlite-server.o: ovsqlite/ovsqlite-server.c ../include/config.h \
+  ../include/inn/defines.h ../include/inn/system.h ../include/inn/macros.h \
+  ../include/inn/portable-macros.h ../include/inn/portable-stdbool.h \
+  ../include/inn/defines.h ../include/inn/options.h \
+  ../include/inn/messages.h ovsqlite/ovsqlite-private.h \
+  ../include/clibrary.h ../include/config.h ../include/inn/macros.h \
+  ../include/portable/stdbool.h ../include/portable/macros.h \
+  ../include/inn/buffer.h ../include/portable/setproctitle.h \
+  ../include/portable/socket.h ../include/portable/getaddrinfo.h \
+  ../include/portable/getnameinfo.h ../include/portable/socket-unix.h \
+  ../include/inn/libinn.h ../include/inn/concat.h ../include/inn/xmalloc.h \
+  ../include/inn/xwrite.h ../include/inn/fdflag.h \
+  ../include/inn/portable-socket.h ../include/inn/innconf.h \
+  ../include/inn/confparse.h ../include/inn/storage.h \
+  ../include/inn/options.h ovsqlite/sql-main.h ovsqlite/sqlite-helper.h \
+  ovsqlite/sql-init.h
+ovsqlite/sql-init.o: ovsqlite/sql-init.c ovsqlite/sql-init.h \
+  ovsqlite/sqlite-helper.h ../include/config.h ../include/inn/defines.h \
+  ../include/inn/system.h ../include/inn/macros.h \
+  ../include/inn/portable-macros.h ../include/inn/portable-stdbool.h \
+  ../include/inn/defines.h ../include/inn/options.h
+ovsqlite/sql-main.o: ovsqlite/sql-main.c ovsqlite/sql-main.h \
+  ovsqlite/sqlite-helper.h ../include/config.h ../include/inn/defines.h \
+  ../include/inn/system.h ../include/inn/macros.h \
+  ../include/inn/portable-macros.h ../include/inn/portable-stdbool.h \
+  ../include/inn/defines.h ../include/inn/options.h
+ovsqlite/sqlite-helper.o: ovsqlite/sqlite-helper.c ovsqlite/sqlite-helper.h \
+  ../include/config.h ../include/inn/defines.h ../include/inn/system.h \
+  ../include/inn/macros.h ../include/inn/portable-macros.h \
+  ../include/inn/portable-stdbool.h ../include/inn/defines.h \
+  ../include/inn/options.h
 tradindexed/tdx-util.o: tradindexed/tdx-util.c ../include/config.h \
   ../include/inn/defines.h ../include/inn/system.h ../include/inn/macros.h \
   ../include/inn/portable-macros.h ../include/inn/portable-stdbool.h \

Index: trunk/storage/ovsqlite
===================================================================
--- storage/ovsqlite	2020-12-25 09:51:06 UTC (rev 10468)
+++ storage/ovsqlite	2020-12-25 09:57:59 UTC (rev 10469)

Property changes on: trunk/storage/ovsqlite
___________________________________________________________________
Added: svn:ignore
## -0,0 +1,6 ##
+ovsqlite-server
+sql-init.c
+sql-init.h
+sql-main.c
+sql-main.h
+sqlite-helper-gen
Added: storage/ovsqlite/ovmethod.config
===================================================================
--- storage/ovsqlite/ovmethod.config	                        (rev 0)
+++ storage/ovsqlite/ovmethod.config	2020-12-25 09:57:59 UTC (rev 10469)
@@ -0,0 +1,7 @@
+name          = ovsqlite
+number        = 5
+sources       = ovsqlite.c ovsqlite-private.c
+extra-sources = ovsqlite-server.c sql-main.c sql-init.c sqlite-helper.c
+programs      = ovsqlite-server
+clean         = sqlite-helper-gen
+maintclean    = sql-init.c sql-init.h sql-main.c sql-main.h


Property changes on: trunk/storage/ovsqlite/ovmethod.config
___________________________________________________________________
Added: svn:eol-style
## -0,0 +1 ##
+native
\ No newline at end of property
Added: svn:keywords
## -0,0 +1 ##
+Author Date Id Revision
\ No newline at end of property
Added: storage/ovsqlite/ovmethod.mk
===================================================================
--- storage/ovsqlite/ovmethod.mk	                        (rev 0)
+++ storage/ovsqlite/ovmethod.mk	2020-12-25 09:57:59 UTC (rev 10469)
@@ -0,0 +1,19 @@
+ovsqlite/ovsqlite-server: ovsqlite/ovsqlite-server.o ovsqlite/sql-main.o \
+		ovsqlite/sql-init.o ovsqlite/sqlite-helper.o \
+		ovsqlite/ovsqlite-private.o
+	$(LIBLD) $(LDFLAGS) $(SQLITE3_LDFLAGS) -o $@ $^ $(LIBSTORAGE) \
+		$(LIBHIST) $(LIBINN) $(STORAGE_LIBS) $(SQLITE3_LIBS) $(LIBS)
+
+ovsqlite/sqlite-helper-gen: ovsqlite/sqlite-helper-gen.in $(FIXSCRIPT)
+	$(FIXSCRIPT) -i ovsqlite/sqlite-helper-gen.in
+
+ovsqlite/sql-main.c: ovsqlite/sql-main.sql ovsqlite/sqlite-helper-gen
+	ovsqlite/sqlite-helper-gen ovsqlite/sql-main.sql
+
+ovsqlite/sql-main.h: ovsqlite/sql-main.c ;
+
+ovsqlite/sql-init.c: ovsqlite/sql-init.sql ovsqlite/sqlite-helper-gen
+	ovsqlite/sqlite-helper-gen ovsqlite/sql-init.sql
+
+ovsqlite/sql-init.h: ovsqlite/sql-init.c ;
+


Property changes on: trunk/storage/ovsqlite/ovmethod.mk
___________________________________________________________________
Added: svn:eol-style
## -0,0 +1 ##
+native
\ No newline at end of property
Added: svn:keywords
## -0,0 +1 ##
+Author Date Id Revision
\ No newline at end of property
Added: storage/ovsqlite/ovsqlite-private.c
===================================================================
--- storage/ovsqlite/ovsqlite-private.c	                        (rev 0)
+++ storage/ovsqlite/ovsqlite-private.c	2020-12-25 09:57:59 UTC (rev 10469)
@@ -0,0 +1,65 @@
+/*  $Id$
+*/
+
+#include "ovsqlite-private.h"
+
+#ifdef HAVE_SQLITE3
+
+#include <string.h>
+#include "inn/xmalloc.h"
+
+bool unpack_now(
+    buffer_t *src,
+    void *bytes,
+    size_t count)
+{
+    if (count>src->left)
+        return false;
+    if (bytes && count>0)
+        memcpy(bytes, src->data+src->used, count);
+    src->used += count;
+    src->left -= count;
+    return true;
+}
+
+void *unpack_later(
+    buffer_t *src,
+    size_t count)
+{
+    void *result;
+
+    if (count>src->left)
+        return NULL;
+    result = src->data+src->used;
+    src->used += count;
+    src->left -= count;
+    return result;
+}
+
+size_t pack_now(
+    buffer_t *dst,
+    void const *bytes,
+    size_t count)
+{
+    if (bytes) {
+        buffer_append(dst, bytes, count);
+    } else {
+        buffer_resize(dst, dst->used+dst->left+count);
+        dst->left += count;
+    }
+    return dst->left;
+}
+
+size_t pack_later(
+    buffer_t *dst,
+    size_t count)
+{
+    size_t result;
+
+    result = dst->left;
+    buffer_resize(dst, dst->used+result+count);
+    dst->left = result+count;
+    return result;
+}
+
+#endif /* HAVE_SQLITE3 */


Property changes on: trunk/storage/ovsqlite/ovsqlite-private.c
___________________________________________________________________
Added: svn:eol-style
## -0,0 +1 ##
+native
\ No newline at end of property
Added: svn:keywords
## -0,0 +1 ##
+Author Date Id Revision
\ No newline at end of property
Added: storage/ovsqlite/ovsqlite-private.h
===================================================================
--- storage/ovsqlite/ovsqlite-private.h	                        (rev 0)
+++ storage/ovsqlite/ovsqlite-private.h	2020-12-25 09:57:59 UTC (rev 10469)
@@ -0,0 +1,484 @@
+/*  $Id$
+*/
+
+#ifndef OVSQLITE_PRIVATE_H
+#define OVSQLITE_PRIVATE_H 1
+
+#include "config.h"
+#include "clibrary.h"
+
+#ifdef HAVE_SQLITE3
+
+#include "portable/macros.h"
+#include "inn/buffer.h"
+
+#define OVSQLITE_SCHEMA_VERSION         1
+#define OVSQLITE_PROTOCOL_VERSION       1
+
+#define OVSQLITE_SERVER_SOCKET          "ovsqlite.sock"
+#define OVSQLITE_SERVER_PIDFILE         "ovsqlite.pid"
+
+#ifndef HAVE_UNIX_DOMAIN_SOCKETS
+
+#define OVSQLITE_SERVER_PORT            "ovsqlite.port"
+
+#define OVSQLITE_COOKIE_LENGTH          16
+
+typedef struct ovsqlite_port {
+    uint16_t port;                      /* in network byte order */
+    uint8_t cookie[OVSQLITE_COOKIE_LENGTH];
+} ovsqlite_port;
+
+#endif /* ! HAVE_UNIX_DOMAIN_SOCKETS */
+
+/*
+ * This needs to stay in sync with the dispatch array
+ * in ovsqlite-server.c or things will explode.
+ */
+
+enum {
+    request_hello,
+    request_set_cutofflow,
+    request_add_group,
+    request_get_groupinfo,
+    request_delete_group,
+    request_list_groups,
+    request_add_article,
+    request_get_artinfo,
+    request_delete_article,
+    request_search_group,
+    request_start_expire_group,
+    request_expire_group,
+    request_finish_expire,
+
+    count_request_codes
+};
+
+enum {
+    response_ok                         = 0x00,
+    response_done,
+    response_groupinfo,
+    response_grouplist,
+    response_grouplist_done,
+    response_artinfo,
+    response_artlist,
+    response_artlist_done,
+
+    response_error                      = 0x80,
+    response_sequence_error,
+    response_sql_error,
+    response_corrupted,
+    response_no_group,
+    response_no_article,
+    response_dup_article,
+    response_old_article,
+
+    response_fatal                      = 0xC0,
+    response_bad_request,
+    response_oversized,
+    response_wrong_state,
+    response_wrong_version,
+    response_failed_auth
+};
+
+enum {
+    search_flag_high                    = 0x01,
+
+    search_flags_all                    = 0x01
+};
+
+enum {
+    search_col_arrived                  = 0x01,
+    search_col_expires                  = 0x02,
+    search_col_token                    = 0x04,
+    search_col_overview                 = 0x08,
+
+    search_cols_all                     = 0x0F
+};
+
+typedef struct buffer buffer_t;
+
+BEGIN_DECLS
+
+extern bool unpack_now(
+    buffer_t *src,
+    void *bytes,
+    size_t count);
+
+extern void *unpack_later(
+    buffer_t *src,
+    size_t count);
+
+extern size_t pack_now(
+    buffer_t *dst,
+    void const *bytes,
+    size_t count);
+
+extern size_t pack_later(
+    buffer_t *dst,
+    size_t count);
+
+END_DECLS
+
+#endif /* HAVE_SQLITE3 */
+
+#endif /* ! OVSQLITE_PRIVATE_H */
+
+
+/****************************************************************************
+
+ovsqlite-server protocol version 1
+
+The protocol is binary and uses no alignment padding anywhere.
+All integer values are in native byte order.
+This description uses "u8", "u16", etc. for unsigned integer values
+and "s8", "s16", etc. for signed integer values.
+Repeat counts are given in brackets.
+Braces specify a group of fields to be repeated.
+
+Each request starts with a u32 specifying the total length in bytes
+(including the length itself) and a u8 containing the request code.
+
+Each response starts with a u32 specifying the total length in bytes
+(including the length itself) and a u8 containing the response code.
+
+The server sends exactly one response for each received request.
+
+Success responses use codes less than 0x80.
+Error responses use codes from 0x80 and up.
+Fatal error responses use codes from 0xC0 and up.
+The server closes the connection after sending a fatal error response.
+
+
+=== request formats ===
+
+request_hello
+    u32 length
+    u8 code
+    u32 version
+    u32 mode
+    u8 cookie[16]   (conditional)
+
+This must be sent as the first (and only the first) request.
+The version field specifies the expected protocol version.
+The mode field contains the read/write flags.
+The cookie field is present only when running on a system without
+Unix-domain sockets and contains the 16-byte authentication cookie.
+
+
+request_setcutofflow
+    u32 length
+    u8 code
+    u8 cutofflow
+
+Used by makehistory to pass along its -I option (defining whether
+overview data for articles numbered lower than the lowest article
+number in active should be stored).
+
+
+request_add_group
+    u32 length
+    u8 code
+    u16 groupname_len
+    u8 groupname[groupname_len]
+    u64 low
+    u64 high
+    u16 flag_alias_len
+    u8 flag_alias[flag_alias_len]
+
+Adds a new group or changes the flag of an existing group.
+
+
+request_get_groupinfo
+    u32 length
+    u8 code
+    u16 groupname_len
+    u8 groupname[groupname_len]
+
+Returns response_groupinfo on success.
+
+
+request_delete_group
+    u32 length
+    u8 code
+    u16 groupname_len
+    u8 groupname[groupname_len]
+
+
+request_list_groups
+    u32 length
+    u8 code
+    u32 space
+    s64 groupid
+
+The space field specifies the largest response size wanted.
+The groupid field specifies where to resume iteration and should be set
+to 0 the first time around.
+Returns response_grouplist or response_grouplist_done on success.
+
+
+request_add_article
+    u32 length
+    u8 code
+    u16 groupname_len
+    u8 groupname[groupname_len]
+    u64 artnum
+    s64 arrived
+    s64 expires
+    u8 token[18]
+    u32 overview_len
+    u8 overview[overview_len]
+
+
+request_get_artinfo
+    u32 length
+    u8 code
+    u16 groupname_len
+    u8 groupname[groupname_len]
+    u64 artnum
+
+Returns response_artinfo on success.
+
+
+request_delete_article
+    u32 length
+    u8 code
+    u16 groupname_len
+    u8 groupname[groupname_len]
+    u64 artnum
+
+
+request_search_group
+    u32 length
+    u8 code
+    u32 space
+    u8 flags
+    u8 cols
+    u16 groupname_len
+    u8 groupname[groupname_len]
+    u64 low
+    u64 high    (optional: search_flag_high)
+
+The space field specifies the largest response size wanted.
+The flags field specifies what optional fields are present
+The cols field specifies what optional columns should be returned.
+
+
+request_start_expire_group
+    u32 length
+    u8 code
+    u16 groupname_len
+    u8 groupname[groupname_len]
+
+Sets the expire timestamp on the group to show that it hasn't been forgotten.
+
+
+request_expire_group
+    u32 length
+    u8 code
+    u16 groupname_len
+    u8 groupname[groupname_len]
+    u32 count
+    u64 artnum[count]
+
+Deletes multiple articles from a group.
+
+
+request_finish_expire
+    u32 length
+    u8 code
+
+Performs clean-up of deleted groups.  In order not to monopolise the server,
+it only does a limited amount of work each time.  The client should repeat
+the request until it receives a response_done.
+
+
+=== response formats ===
+
+response_ok
+    u32 length
+    u8 code
+
+The generic success response.
+
+
+response_done
+    u32 length
+    u8 code
+
+Returned from request_finish_expire when there is no more work to be done.
+
+
+response_groupinfo
+    u32 length
+    u8 code
+    u64 low
+    u64 high
+    u64 count
+    u16 flag_alias_len
+    u8 flag_alias[flag_alias_len]
+
+Returned from request_get_groupinfo.
+
+
+response_grouplist
+response_grouplist_done
+    u32 length
+    u8 code
+    s64 groupid
+    u32 count
+    {
+        u16 groupname_len
+        u8 groupname[groupname_len]
+        u64 low
+        u64 high
+        u64 count
+        u16 flag_alias_len
+        u8 flag_alias[flag_alias_len]
+    } [count]
+
+Returned from request_list_groups.
+The groupid field should be copied to the next request_list_groups
+to get the next batch of groups.
+The code response_grouplist_done means that this is the final batch
+of groups and further requests are pointless.
+
+
+response_artinfo
+    u32 length
+    u8 code
+    u8 token[18]
+
+Returned from request_get_artinfo.
+
+
+response_artlist
+response_artlist_done
+    u32 length
+    u8 code
+    u8 cols
+    u32 count
+    {
+        u64 artnum
+        s64 arrived                 (optional: search_col_arrived)
+        s64 expires                 (optional: search_col_expires)
+        u8 token[18]                (optional: search_col_token)
+        u32 overview_len            (optional: search_col_overview)
+        u8 overview[overview_len]   (optional: search_col_overview)
+    } [count]
+
+Returned from request_search_group.
+The cols field specifies what optional fields are included.  It is copied
+from the request.
+The code response_artlist_done means that this is the last batch
+of articles and further requests are pointless.
+
+
+response_error
+    u32 length
+    u8 code
+
+A generic error response with no further information.
+
+
+response_sequence_error
+    u32 length
+    u8 code
+
+Returned when the client sends requests in the wrong order, for example
+request_finish_expire not preceded by at least one request_start_expire_group.
+
+
+response_sql_error
+    u32 length
+    u8 code
+    s32 status
+    u32 errmsg_len
+    u8 errmsg[errmsg_len]
+
+Returned when an unexpected database error occurs.
+The status and errmsg fields come directly from SQLite.
+
+
+response_corrupted
+    u32 length
+    u8 code
+
+Returned when incorrect or inconsistent data is found in the database.
+
+
+response_no_group
+    u32 length
+    u8 code
+
+Returned when a request contains an unknown group name.
+
+
+response_no_article
+    u32 length
+    u8 code
+
+Returned when a request contains an unknown article number.
+
+
+response_dup_article
+    u32 length
+    u8 code
+
+Returned from request_add_article when an article with the specified
+group name and article number already exists.
+
+
+response_old_article
+    u32 length
+    u8 code
+
+Returned from request_add_article when the article number is less
+than the group lowmark and cutofflow has been set to true.
+
+
+response_fatal
+    u32 length
+    u8 code
+
+A generic fatal error response with no further information.
+
+
+response_bad_request
+    u32 length
+    u8 code
+
+Returned from a request with an unknown code or the wrong format.
+
+
+response_oversized
+    u32 length
+    u8 code
+
+Returned from a request with an unreasonably large size.
+
+
+response_wrong_state
+    u32 length
+    u8 code
+
+Returned when the first request isn't a request_hello
+or when a request_hello isn't the first request.
+
+
+response_wrong_version
+    u32 length
+    u8 code
+
+Returned from a request_hello with the wrong version.
+
+
+response_failed_auth
+    u32 length
+    u8 code
+
+Returned from a request_hello with the wrong cookie.
+
+
+****************************************************************************/
+


Property changes on: trunk/storage/ovsqlite/ovsqlite-private.h
___________________________________________________________________
Added: svn:eol-style
## -0,0 +1 ##
+native
\ No newline at end of property
Added: svn:keywords
## -0,0 +1 ##
+Author Date Id Revision
\ No newline at end of property
Added: storage/ovsqlite/ovsqlite-server.c
===================================================================
--- storage/ovsqlite/ovsqlite-server.c	                        (rev 0)
+++ storage/ovsqlite/ovsqlite-server.c	2020-12-25 09:57:59 UTC (rev 10469)
@@ -0,0 +1,2174 @@
+/*  $Id$
+**
+**  Daemon server to access overview database based on SQLite.
+**
+**  Original implementation written by Bo Lindbergh (2020-12-17).
+**  <2bfjdsla52kztwejndzdstsxl9athp at gmail.com>
+*/
+
+#include "config.h"
+#include "inn/messages.h"
+#include "ovsqlite-private.h"
+
+#ifdef HAVE_SQLITE3
+
+#include <errno.h>
+#include <signal.h>
+#include <fcntl.h>
+#include <syslog.h>
+#ifdef HAVE_SYS_SELECT_H
+# include <sys/select.h>
+#endif
+#include <time.h>
+#ifdef HAVE_SYS_TIME_H
+# include <sys/time.h>
+#endif
+#include <sys/stat.h>
+
+#include "portable/setproctitle.h"
+#include "portable/socket.h"
+#ifdef HAVE_UNIX_DOMAIN_SOCKETS
+# include "portable/socket-unix.h"
+#endif
+
+#include "inn/libinn.h"
+#include "inn/concat.h"
+#include "inn/fdflag.h"
+#include "inn/xmalloc.h"
+#include "inn/innconf.h"
+#include "inn/confparse.h"
+#include "inn/storage.h"
+
+#include "sql-main.h"
+#include "sql-init.h"
+
+#define OVSQLITE_DB_FILE "ovsqlite.db"
+
+#ifdef HAVE_ZLIB
+
+#define USE_DICTIONARY 1
+
+#include <zlib.h>
+
+static z_stream deflation;
+static z_stream inflation;
+
+static buffer_t *flate;
+
+static uint32_t const pack_length_bias[5] =
+{
+             0,
+          0x80,
+        0x4080,
+      0x204080,
+    0x10204080,
+};
+
+#ifdef USE_DICTIONARY
+
+static char const basedict_format[] =
+    "\tRe: =?UTF-8?Q? =?UTF-8?B? the The and for "
+    "\tMon, \tTue, \tWed, \tThu, \tFri, \tSat, \tSun, "
+    "Jan Feb Mar Apr May Jun Jul Aug Sep Oct Nov Dec "
+    "GMT\t (UTC)\tXref: %s ";
+
+static char dictionary[0x8000];
+static unsigned int basedict_len;
+
+#endif /* USE_DICTIONARY */
+
+#endif /* HAVE_ZLIB */
+
+enum {
+    client_flag_init    = 0x01,
+    client_flag_term    = 0x02,
+};
+
+#define INITIAL_CAPACITY 0x400
+
+typedef struct client_t {
+    uint8_t flags;
+    bool cutofflow;
+    bool check_forgotten;
+    int sock;
+    uint32_t mode;
+    time_t expiration_start;
+    buffer_t *request;
+    buffer_t *response;
+} client_t;
+
+#ifndef HAVE_UNIX_DOMAIN_SOCKETS
+static ovsqlite_port port;
+#endif /* ! HAVE_UNIX_DOMAIN_SOCKETS */
+
+static char *pidfile = NULL;
+static int listensock = -1;
+static int maxsock = -1;
+static client_t *clients = NULL;
+static size_t client_capacity, client_count;
+static fd_set read_fds, write_fds;
+static bool volatile terminating;
+
+static sqlite3 *connection;
+static sql_main_t sql_main;
+
+static bool use_compression;
+static unsigned long pagesize;
+static unsigned long cachesize;
+static struct timeval transaction_time_limit = {10, 0};
+static unsigned long transaction_row_limit = 10000;
+
+static bool in_transaction;
+static unsigned int transaction_rowcount;
+static struct timeval next_commit;
+
+
+static void timeval_normalise(
+    struct timeval *t)
+{
+    time_t carry;
+
+    carry = t->tv_usec/1000000;
+    if (carry) {
+        t->tv_sec += carry;
+        t->tv_usec -= carry*1000000;
+    }
+    if (t->tv_sec>0 && t->tv_usec<0) {
+        t->tv_sec--;
+        t->tv_usec += 1000000;
+    } else if (t->tv_sec<0 && t->tv_usec>0) {
+        t->tv_sec++;
+        t->tv_usec -= 1000000;
+    }
+}
+
+static struct timeval timeval_sum(
+    struct timeval a,
+    struct timeval b)
+{
+    struct timeval result;
+
+    timeval_normalise(&a);
+    timeval_normalise(&b);
+    result.tv_sec = a.tv_sec+b.tv_sec;
+    result.tv_usec = a.tv_usec+b.tv_usec;
+    timeval_normalise(&result);
+    return result;
+}
+
+static struct timeval timeval_difference(
+    struct timeval a,
+    struct timeval b)
+{
+    struct timeval result;
+
+    timeval_normalise(&a);
+    timeval_normalise(&b);
+    result.tv_sec = a.tv_sec-b.tv_sec;
+    result.tv_usec = a.tv_usec-b.tv_usec;
+    timeval_normalise(&result);
+    return result;
+}
+
+static void catcher(
+    int sig UNUSED)
+{
+    terminating = true;
+}
+
+static void catch_signals(void)
+{
+    xsignal_norestart(SIGINT, catcher);
+    xsignal_norestart(SIGTERM, catcher);
+    xsignal_norestart(SIGHUP, catcher);
+    xsignal(SIGPIPE, SIG_IGN);
+}
+
+static void resetclear(
+    sqlite3_stmt *stmt)
+{
+    sqlite3_reset(stmt);
+    sqlite3_clear_bindings(stmt);
+}
+
+static client_t *add_client(
+    int sock)
+{
+    client_t *result;
+
+    if (client_count>=client_capacity) {
+        size_t new_client_capacity;
+
+        new_client_capacity = (client_count+1)*3/2;
+        clients = xreallocarray(
+            clients, new_client_capacity, sizeof (client_t));
+        client_capacity = new_client_capacity;
+    }
+
+    result = clients+client_count;
+    memset(result, 0, sizeof (client_t));
+    result->sock = sock;
+    result->flags = client_flag_init;
+    result->request = buffer_new();
+    buffer_resize(result->request, INITIAL_CAPACITY);
+    result->response = buffer_new();
+    buffer_resize(result->response, INITIAL_CAPACITY);
+
+    client_count++;
+    FD_SET(sock, &read_fds);
+    if (sock>maxsock)
+        maxsock = sock;
+    return result;
+}
+
+static void del_client(
+    client_t *client)
+{
+    size_t ix;
+    int sock;
+
+    ix = client-clients;
+    if (ix>=client_count || client!=clients+ix)
+        return;
+    sock = client->sock;
+    FD_CLR(sock, &read_fds);
+    FD_CLR(sock, &write_fds);
+    close(sock);
+    buffer_free(client->request);
+    buffer_free(client->response);
+    if (ix+1<client_count)
+        *client = clients[client_count-1];
+    client_count--;
+    if (sock==maxsock) {
+        int new_maxsock;
+
+        new_maxsock = listensock;
+        for (ix = 0; ix<client_count; ix++) {
+            sock = clients[ix].sock;
+            if (sock>new_maxsock)
+                new_maxsock = sock;
+        }
+        maxsock = new_maxsock;
+    }
+}
+
+#ifdef HAVE_UNIX_DOMAIN_SOCKETS
+
+static void make_unix_listener(void)
+{
+    char *path;
+    struct sockaddr_un sa;
+
+    listensock = socket(AF_UNIX, SOCK_STREAM, 0);
+    if (listensock==-1)
+        sysdie("cannot create socket");
+    memset(&sa, 0, sizeof sa);
+    sa.sun_family = AF_UNIX;
+    path = concatpath(innconf->pathrun, OVSQLITE_SERVER_SOCKET);
+    strlcpy(sa.sun_path, path, sizeof(sa.sun_path));
+    unlink(sa.sun_path);
+    free(path);
+    if (bind(listensock, (struct sockaddr *)&sa, SUN_LEN(&sa))!=0)
+        sysdie("cannot bind socket");
+}
+
+#else /* ! HAVE_UNIX_DOMAIN_SOCKETS */
+
+static void make_inet_listener(void)
+{
+    char *path;
+    struct sockaddr_in sa;
+    int status;
+    void const *cookie;
+    int ret;
+    socklen_t salen;
+    int fd;
+
+    sqlite3_bind_int(sql_main.random, 1, OVSQLITE_COOKIE_LENGTH);
+    status = sqlite3_step(sql_main.random);
+    if (status!=SQLITE_ROW) {
+        die("SQLite error while generating random cookie: %s",
+            sqlite3_errmsg(connection));
+    }
+    cookie = sqlite3_column_blob(sql_main.random, 0);
+    if (!cookie) {
+        status = sqlite3_errcode(connection);
+        if (status!=SQLITE_OK) {
+            die("SQLite error while generating random cookie: %s",
+                sqlite3_errmsg(connection));
+        } else {
+            die("unexpected NULL result while generating random cookie");
+        }
+    }
+    if (sqlite3_column_bytes(sql_main.random, 0)!=OVSQLITE_COOKIE_LENGTH)
+        die("unexpected result size while generating random cookie");
+    memcpy(port.cookie, cookie, OVSQLITE_COOKIE_LENGTH);
+    resetclear(sql_main.random);
+
+    listensock = socket(AF_INET, SOCK_STREAM, 0);
+    if (listensock==-1)
+        sysdie("cannot create socket");
+    memset(&sa, 0, sizeof sa);
+    sa.sin_family = AF_INET;
+    sa.sin_addr.s_addr = htonl(0x7f000001UL);
+    if (bind(listensock, (struct sockaddr *)&sa, sizeof sa)!=0)
+        sysdie("cannot bind socket");
+    salen = sizeof sa;
+    if (getsockname(listensock, (struct sockaddr *)&sa, &salen)!=0)
+        sysdie("cannot extract socket port number");
+    port.port = sa.sin_port;
+
+    path = concatpath(innconf->pathrun, OVSQLITE_SERVER_PORT);
+    unlink(path);
+    fd = open(path, O_CREAT|O_TRUNC|O_WRONLY, 0440);
+    if (fd==-1)
+        sysdie("cannot create port file %s", path);
+    ret = write(fd, &port, sizeof port);
+    if (ret==-1)
+        sysdie("cannot write port file %s", path);
+    if (ret!=sizeof port)
+        die("cannot write port file %s: Short write", path);
+    if (close(fd)!=0)
+        sysdie("cannot close port file %s", path);
+    free(path);
+}
+
+#endif /* ! HAVE_UNIX_DOMAIN_SOCKETS */
+
+static void make_listener(void)
+{
+#ifdef HAVE_UNIX_DOMAIN_SOCKETS
+    make_unix_listener();
+#else
+    make_inet_listener();
+#endif
+    if (listen(listensock, MAXLISTEN)==-1)
+        sysdie("cannot listen on socket");
+    fdflag_nonblocking(listensock, 1);
+    FD_SET(listensock, &read_fds);
+    maxsock = listensock;
+}
+
+static void make_pidfile(void)
+{
+    FILE *pf;
+
+    pidfile = concatpath(innconf->pathrun, OVSQLITE_SERVER_PIDFILE);
+    pf = fopen(pidfile, "w");
+    if (!pf)
+        sysdie("cannot create PID file");
+    if (fprintf(pf, "%ld\n", (long)getpid())<0)
+        sysdie("cannot write PID file");
+    if (fclose(pf))
+        sysdie("cannot close PID file");
+}
+
+static void load_config(void)
+{
+    char *path;
+    struct config_group *top;
+    double timelimit;
+
+    if (strcmp(innconf->ovmethod, "ovsqlite"))
+        die("ovmethod not set to ovsqlite in inn.conf");
+    path = concatpath(innconf->pathetc, "ovsqlite.conf");
+    top = config_parse_file(path);
+    free(path);
+    if (top) {
+        config_param_boolean(top, "compress", &use_compression);
+        config_param_unsigned_number(
+            top, "pagesize", &pagesize);
+        config_param_unsigned_number(
+            top, "cachesize", &cachesize);
+        if (config_param_real(top, "transtimelimit", &timelimit)) {
+            transaction_time_limit.tv_sec = timelimit;
+            transaction_time_limit.tv_usec =
+                (timelimit-transaction_time_limit.tv_sec)*1E6+0.5;
+            timeval_normalise(&transaction_time_limit);
+        }
+        config_param_unsigned_number(
+            top, "transrowlimit", &transaction_row_limit);
+
+        config_free(top);
+    }
+}
+
+#ifdef HAVE_ZLIB
+
+static void setup_compression(
+    bool init)
+{
+    int status;
+
+    deflation.zalloc = Z_NULL;
+    deflation.zfree = Z_NULL;
+    deflation.opaque = Z_NULL;
+    deflation.next_in = Z_NULL;
+    deflation.avail_in = 0;
+    status = deflateInit(&deflation, Z_DEFAULT_COMPRESSION);
+    if (status!=Z_OK) {
+        if (deflation.msg) {
+            die("cannot set up compression: %s", deflation.msg);
+        } else {
+            die("cannot set up compression");
+        }
+    }
+
+    inflation.zalloc = Z_NULL;
+    inflation.zfree = Z_NULL;
+    inflation.opaque = Z_NULL;
+    inflation.next_in = Z_NULL;
+    inflation.avail_in = 0;
+    status = inflateInit(&inflation);
+    if (status!=Z_OK) {
+        if (inflation.msg) {
+            die("cannot set up decompression: %s", inflation.msg);
+        } else {
+            die("cannot set up decompression");
+        }
+    }
+
+#ifdef USE_DICTIONARY
+    if (init) {
+        basedict_len = snprintf(
+            dictionary, sizeof dictionary,
+            basedict_format, innconf->pathhost);
+        sqlite3_bind_text(sql_main.setmisc, 1, "basedict", -1, SQLITE_STATIC);
+        sqlite3_bind_blob(
+            sql_main.setmisc, 2, dictionary, basedict_len, SQLITE_STATIC);
+        status = sqlite3_step(sql_main.setmisc);
+        if (status!=SQLITE_DONE) {
+            die("cannot store compression dictionary: %s",
+                sqlite3_errmsg(connection));
+        }
+        resetclear(sql_main.setmisc);
+    } else {
+        void const *dict;
+        size_t size;
+
+        sqlite3_bind_text(sql_main.getmisc, 1, "basedict", -1, SQLITE_STATIC);
+        status = sqlite3_step(sql_main.getmisc);
+        if (status!=SQLITE_ROW) {
+            die("cannot load compression dictionary: %s",
+                sqlite3_errmsg(connection));
+        }
+        dict = sqlite3_column_blob(sql_main.getmisc, 0);
+        size = sqlite3_column_bytes(sql_main.getmisc, 0);
+        if (!dict || size>=sizeof dictionary)
+            die("invalid compression dictionary in database");
+        memcpy(dictionary, dict, size);
+        basedict_len = size;
+        resetclear(sql_main.getmisc);
+    }
+#endif
+
+    flate = buffer_new();
+}
+
+static void pack_length(
+    z_stream *stream,
+    uint32_t length)
+{
+    unsigned int lenlen, n;
+    uint8_t *walk;
+
+    lenlen = 1;
+    while (lenlen<5 && length>=pack_length_bias[lenlen])
+        lenlen++;
+    length -= pack_length_bias[lenlen-1];
+    if (stream->avail_out<lenlen) {
+        die("BUG!  pack_length called with insufficient buffer space (%u<%u)",
+            stream->avail_out,lenlen);
+    }
+    walk = stream->next_out+lenlen;
+    stream->next_out = walk;
+    stream->avail_out -= lenlen;
+    for (n = lenlen; n>1; n--) {
+        *--walk = length;
+        length >>= 8;
+    }
+    *--walk = length | (~0U<<(9-lenlen));
+}
+
+static uint32_t unpack_length(
+    z_stream *stream)
+{
+    uint8_t *walk;
+    unsigned int c, lenlen, n;
+    uint32_t length;
+
+    if (stream->avail_in<=0)
+        die("BUG!  unpack_length called with empty buffer");
+    walk = stream->next_in;
+    c = *walk++;
+    lenlen = 1;
+    while (c & (1U<<(8-lenlen)))
+        lenlen++;
+    if (lenlen>5 || lenlen>stream->avail_in)
+        return ~0U;
+    length = c&~(~0U<<(8-lenlen));
+    for (n=lenlen-1; n>0; n--)
+        length = (length<<8) | *walk++;
+    length += pack_length_bias[lenlen-1];
+    stream->next_in = walk;
+    stream->avail_in -= lenlen;
+    return length;
+}
+
+#ifdef USE_DICTIONARY
+
+static unsigned int make_dict(
+    char const *groupname,
+    int groupname_len,
+    uint64_t artnum)
+{
+    sqlite3_snprintf(
+        sizeof dictionary-basedict_len, dictionary+basedict_len,
+        "%.*s:%llu\r\n",
+        groupname_len, groupname, artnum);
+    return basedict_len+strlen(dictionary+basedict_len);
+}
+
+
+#endif
+
+#endif /* HAVE_ZLIB */
+
+static void open_db(void)
+{
+    char *path;
+    struct stat sb;
+    int status;
+    char *errmsg;
+    bool init;
+    char sqltext[64];
+
+    path = concatpath(innconf->pathoverview, OVSQLITE_DB_FILE);
+    init = stat(path, &sb)==-1;
+    if (init) {
+        sql_init_t sql_init;
+
+        status = sqlite3_open_v2(
+            path,
+            &connection,
+            SQLITE_OPEN_READWRITE | SQLITE_OPEN_CREATE,
+            NULL);
+        if (status != SQLITE_OK)
+            die("cannot create database: %s", sqlite3_errstr(status));
+        sqlite3_extended_result_codes(connection, 1);
+        status = sqlite_helper_init(
+            &sql_init_helper,
+            (sqlite3_stmt **)&sql_init,
+            connection,
+            0,
+            &errmsg);
+        if (status != SQLITE_OK)
+            die("cannot initialize new database: %s", errmsg);
+        if (pagesize) {
+            unsigned long default_pagesize;
+
+            status = sqlite3_step(sql_init.getpagesize);
+            if (status != SQLITE_ROW)
+                die("cannot get pagesize: %s", sqlite3_errmsg(connection));
+            default_pagesize = sqlite3_column_int64(sql_init.getpagesize, 0);
+            sqlite3_reset(sql_init.getpagesize);
+            if (default_pagesize!=pagesize) {
+                /* can't use placeholders for pragma arguments, alas */
+                snprintf(sqltext, sizeof sqltext,
+                         "pragma page_size=%lu; vacuum;",
+                         pagesize);
+                status = sqlite3_exec(connection, sqltext, 0, NULL, &errmsg);
+                if (status!=SQLITE_OK)
+                    die("cannot set pagesize: %s", errmsg);
+            }
+        }
+        sqlite_helper_term(&sql_init_helper, (sqlite3_stmt **)&sql_init);
+    } else {
+        status = sqlite3_open_v2(
+            path,
+            &connection,
+            SQLITE_OPEN_READWRITE,
+            NULL);
+        if (status != SQLITE_OK)
+            die("cannot open database: %s", sqlite3_errstr(status));
+        sqlite3_extended_result_codes(connection, 1);
+    }
+    status = sqlite_helper_init(
+        &sql_main_helper,
+        (sqlite3_stmt **)&sql_main,
+        connection,
+        SQLITE_PREPARE_PERSISTENT,
+        &errmsg);
+    if (status!=SQLITE_OK)
+        die("cannot set up database session: %s", errmsg);
+    if (init) {
+        sqlite3_bind_text(sql_main.setmisc, 1, "version", -1, SQLITE_STATIC);
+        sqlite3_bind_int64(sql_main.setmisc, 2, OVSQLITE_SCHEMA_VERSION);
+        status = sqlite3_step(sql_main.setmisc);
+        if (status!=SQLITE_DONE)
+            die("cannot initialize new database: %s",
+                sqlite3_errmsg(connection));
+        resetclear(sql_main.setmisc);
+
+#ifndef HAVE_ZLIB
+        if (use_compression) {
+            warn("compression requested but INN was not built with zlib");
+            use_compression = false;
+        }
+#endif /* ! HAVE_ZLIB */
+        sqlite3_bind_text(sql_main.setmisc, 1, "compress", -1, SQLITE_STATIC);
+        sqlite3_bind_int(sql_main.setmisc, 2, use_compression);
+        status = sqlite3_step(sql_main.setmisc);
+        if (status!=SQLITE_DONE)
+            die("cannot initialize new database: %s",
+                sqlite3_errmsg(connection));
+        resetclear(sql_main.setmisc);
+    } else {
+        int version;
+
+        sqlite3_bind_text(sql_main.getmisc, 1, "version", -1, SQLITE_STATIC);
+        status = sqlite3_step(sql_main.getmisc);
+        if (status!=SQLITE_ROW)
+            die("cannot set up database session: %s",
+                sqlite3_errmsg(connection));
+        version = sqlite3_column_int(sql_main.getmisc, 0);
+        if (version!=OVSQLITE_SCHEMA_VERSION)
+            die("incompatible database schema %d", version);
+        resetclear(sql_main.getmisc);
+
+        sqlite3_bind_text(sql_main.getmisc, 1, "compress", -1, SQLITE_STATIC);
+        status = sqlite3_step(sql_main.getmisc);
+        if (status!=SQLITE_ROW)
+            die("cannot set up database session: %s",
+                sqlite3_errmsg(connection));
+        use_compression = sqlite3_column_int(sql_main.getmisc, 0);
+#ifndef HAVE_ZLIB
+        if (use_compression)
+            die("database uses compression but INN was not built with zlib");
+#endif /* ! HAVE_ZLIB */
+        resetclear(sql_main.getmisc);
+    }
+#ifdef HAVE_ZLIB
+    if (use_compression)
+        setup_compression(init);
+#endif
+    if (cachesize) {
+        snprintf(
+            sqltext, sizeof sqltext,
+            "pragma cache_size = -%lu;",
+            cachesize);
+        status = sqlite3_exec(connection, sqltext, 0, NULL, &errmsg);
+        if (status!=SQLITE_OK) {
+            warn("cannot set cache size: %s", errmsg);
+            sqlite3_free(errmsg);
+        }
+    }
+}
+
+static void close_db(void)
+{
+    sqlite3_step(sql_main.delete_journal);
+    sqlite3_reset(sql_main.delete_journal);
+    sqlite_helper_term(&sql_main_helper, (sqlite3_stmt **)&sql_main);
+    sqlite3_close_v2(connection);
+    connection = NULL;
+#ifdef HAVE_ZLIB
+    if (use_compression) {
+        inflateEnd(&inflation);
+        deflateEnd(&deflation);
+        buffer_free(flate);
+        flate = NULL;
+    }
+#endif
+}
+
+static void close_sockets(void)
+{
+    size_t ix;
+
+    close(listensock);
+    listensock = -1;
+    for (ix = client_count; ix-->0; )
+        del_client(clients+ix);
+}
+
+static unsigned int start_request(
+    client_t *client)
+{
+    uint8_t code;
+
+    unpack_later(client->request, 4);
+    unpack_now(client->request, &code, sizeof code);
+    return code;
+}
+
+static bool finish_request(
+    client_t *client)
+{
+    return client->request->left==0;
+}
+
+static void start_response(
+    client_t *client,
+    unsigned int code)
+{
+    uint8_t code_r;
+
+    buffer_set(client->response, NULL, 0);
+    code_r = code;
+    pack_later(client->response, 4);
+    pack_now(client->response, &code_r, sizeof code_r);
+}
+
+static void finish_response(
+    client_t *client)
+{
+    *(uint32_t *)(void *)client->response->data = client->response->left;
+    FD_SET(client->sock, &write_fds);
+}
+
+static void simple_response(
+    client_t *client,
+    unsigned int code)
+{
+    start_response(client, code);
+    finish_response(client);
+    if (code>=response_fatal) {
+        client->flags |= client_flag_term;
+        FD_CLR(client->sock, &read_fds);
+    }
+}
+
+static void begin_transaction(void)
+{
+    struct timeval now;
+
+    if (in_transaction)
+        return;
+    gettimeofday(&now, NULL);
+    next_commit = timeval_sum(now, transaction_time_limit);
+    sqlite3_step(sql_main.begin);
+    sqlite3_reset(sql_main.begin);
+    in_transaction = true;
+}
+
+static void commit_transaction(void)
+{
+    if (!in_transaction)
+        return;
+    sqlite3_step(sql_main.commit);
+    sqlite3_reset(sql_main.commit);
+    transaction_rowcount = 0;
+    in_transaction = false;
+}
+
+static void savepoint(void)
+{
+    sqlite3_step(sql_main.savepoint);
+    sqlite3_reset(sql_main.savepoint);
+}
+
+static void release_savepoint(void)
+{
+    sqlite3_step(sql_main.release_savepoint);
+    sqlite3_reset(sql_main.release_savepoint);
+}
+
+static void rollback_savepoint(void)
+{
+    sqlite3_step(sql_main.rollback_savepoint);
+    sqlite3_reset(sql_main.rollback_savepoint);
+}
+
+static void sql_error_response(
+    client_t *client,
+    int status)
+{
+    buffer_t *respbuf;
+    int32_t status_r;
+    char const *errmsg;
+    uint32_t errmsg_len;
+
+    respbuf = client->response;
+    status_r = status;
+    errmsg = sqlite3_errmsg(connection);
+    if (errmsg) {
+        errmsg_len = strlen(errmsg);
+    } else {
+        errmsg_len = 0;
+    }
+    start_response(client, response_sql_error);
+    pack_now(respbuf, &status_r, sizeof status_r);
+    pack_now(respbuf, &errmsg_len, sizeof errmsg_len);
+    pack_now(respbuf, errmsg, errmsg_len);
+    finish_response(client);
+}
+
+#define failvar \
+    unsigned int code = response_ok
+
+#define failvar_stmt \
+    failvar; \
+    int status = SQLITE_OK; \
+    sqlite3_stmt *stmt = NULL
+
+#define failvar_stmt_savepoint \
+    failvar_stmt; \
+    bool have_savepoint = false
+
+#define fail(err) do { code = (err); goto failure; } while (false)
+
+#define fail_stmt() goto failure_stmt
+
+#define failure_response \
+    failure: \
+        simple_response(client, code); \
+        break
+
+#define failure_stmt_response \
+    failure_stmt: \
+        sql_error_response(client, status); \
+        break
+
+#define failhandling \
+    do { \
+        break; \
+        failure_response; \
+    } while (0)
+
+#define failhandling_stmt \
+    do { \
+        break; \
+        do { \
+            failure_stmt_response; \
+            failure_response; \
+        } while (0); \
+        if (stmt) \
+            resetclear(stmt); \
+    } while (0)
+
+#define failhandling_stmt_savepoint \
+    do { \
+        break; \
+        do { \
+            failure_stmt_response; \
+            failure_response; \
+        } while (0); \
+        if (stmt) \
+            resetclear(stmt); \
+        if (have_savepoint) { \
+            rollback_savepoint(); \
+            release_savepoint(); \
+        } \
+    } while (0)
+
+static void do_hello(
+    client_t *client)
+{
+    buffer_t *reqbuf;
+    uint32_t version;
+    uint32_t mode;
+#ifndef HAVE_UNIX_DOMAIN_SOCKETS
+    void *cookie;
+#endif
+    failvar;
+
+    reqbuf = client->request;
+    if (!unpack_now(reqbuf, &version, sizeof version))
+        fail(response_bad_request);
+    if (!unpack_now(reqbuf, &mode, sizeof mode))
+        fail(response_bad_request);
+#ifndef HAVE_UNIX_DOMAIN_SOCKETS
+    cookie = unpack_later(reqbuf, OVSQLITE_COOKIE_LENGTH);
+    if (!cookie)
+        fail(response_bad_request);
+#endif
+    if (!finish_request(client))
+        fail(response_bad_request);
+
+    if (version!=OVSQLITE_PROTOCOL_VERSION)
+        fail(response_wrong_version);
+#ifndef HAVE_UNIX_DOMAIN_SOCKETS
+    if (memcmp(cookie, port.cookie, OVSQLITE_COOKIE_LENGTH)!=0)
+        fail(response_failed_auth);
+#endif
+
+    client->flags &= ~client_flag_init;
+    client->mode = mode;
+    simple_response(client, response_ok);
+    return;
+
+    failhandling;
+}
+
+static void do_set_cutofflow(
+    client_t *client)
+{
+    buffer_t *reqbuf;
+    uint8_t cutofflow;
+    failvar;
+
+    reqbuf = client->request;
+    if (!unpack_now(reqbuf, &cutofflow, sizeof cutofflow))
+        fail(response_bad_request);
+    if (!finish_request(client))
+        fail(response_bad_request);
+
+    client->cutofflow = cutofflow;
+    simple_response(client, response_ok);
+    return;
+
+    failhandling;
+}
+
+static void do_add_group(
+    client_t *client)
+{
+    buffer_t *reqbuf;
+    void *groupname;
+    uint16_t groupname_len;
+    void *flag_alias;
+    uint16_t flag_alias_len;
+    uint64_t low, high;
+    int64_t groupid = 0;
+    int changes;
+    failvar_stmt;
+
+    reqbuf = client->request;
+    if (!unpack_now(reqbuf, &groupname_len, sizeof groupname_len))
+        fail(response_bad_request);
+    groupname = unpack_later(reqbuf, groupname_len);
+    if (!groupname)
+        fail(response_bad_request);
+    if (!unpack_now(reqbuf, &low, sizeof low))
+        fail(response_bad_request);
+    if (!unpack_now(reqbuf, &high, sizeof high))
+        fail(response_bad_request);
+    if (!unpack_now(reqbuf, &flag_alias_len, sizeof flag_alias_len))
+        fail(response_bad_request);
+    flag_alias = unpack_later(reqbuf, flag_alias_len);
+    if (!flag_alias)
+        fail(response_bad_request);
+    if (!finish_request(client))
+        fail(response_bad_request);
+
+    begin_transaction();
+
+    stmt = sql_main.lookup_groupinfo;
+    sqlite3_bind_blob(stmt, 1, groupname, groupname_len, SQLITE_STATIC);
+    status = sqlite3_step(stmt);
+    switch (status) {
+    case SQLITE_ROW:
+        groupid = sqlite3_column_int64(stmt, 0);
+        break;
+    case SQLITE_DONE:
+        break;
+    default:
+        fail_stmt();
+    }
+    resetclear(stmt);
+    stmt = NULL;
+
+    if (!groupid) {
+        stmt = sql_main.add_group;
+        sqlite3_bind_blob(
+            stmt, 1, groupname, groupname_len, SQLITE_STATIC);
+        sqlite3_bind_blob(
+            stmt, 2, flag_alias, flag_alias_len, SQLITE_STATIC);
+        sqlite3_bind_int64(stmt, 3, low);
+        sqlite3_bind_int64(stmt, 4, high);
+        status = sqlite3_step(stmt);
+        if (status!=SQLITE_DONE)
+            fail_stmt();
+        changes = sqlite3_changes(connection);
+        resetclear(stmt);
+        stmt = NULL;
+    } else {
+        stmt = sql_main.update_groupinfo_flag_alias;
+        sqlite3_bind_int64(stmt, 1, groupid);
+        sqlite3_bind_blob(
+            stmt, 2, flag_alias, flag_alias_len, SQLITE_STATIC);
+        status = sqlite3_step(stmt);
+        if (status!=SQLITE_DONE)
+            fail_stmt();
+        changes = sqlite3_changes(connection);
+        resetclear(stmt);
+        stmt = NULL;
+    }
+
+    if (changes>0)
+        commit_transaction();
+    simple_response(client, response_ok);
+    return;
+
+    failhandling_stmt;
+}
+
+static void do_get_groupinfo(
+    client_t *client)
+{
+    buffer_t *reqbuf, *respbuf;
+    void *groupname;
+    uint16_t groupname_len;
+    char const *flag_alias;
+    uint16_t flag_alias_len;
+    uint64_t low;
+    uint64_t high;
+    uint64_t count;
+    size_t size;
+    failvar_stmt;
+
+    reqbuf = client->request;
+    if (!unpack_now(reqbuf, &groupname_len, sizeof groupname_len))
+        fail(response_bad_request);
+    groupname = unpack_later(reqbuf, groupname_len);
+    if (!groupname)
+        fail(response_bad_request);
+    if (!finish_request(client))
+        fail(response_bad_request);
+
+    stmt = sql_main.get_groupinfo;
+    sqlite3_bind_blob(stmt, 1, groupname, groupname_len, SQLITE_STATIC);
+    status = sqlite3_step(stmt);
+    switch (status) {
+    case SQLITE_ROW:
+        break;
+    case SQLITE_DONE:
+        fail(response_no_group);
+    default:
+        fail_stmt();
+    }
+    low = sqlite3_column_int64(stmt, 0);
+    high = sqlite3_column_int64(stmt, 1);
+    count = sqlite3_column_int64(stmt, 2);
+    flag_alias = sqlite3_column_blob(stmt, 3);
+    if (!flag_alias)
+        fail(response_corrupted);
+    size = sqlite3_column_bytes(stmt, 3);
+    if (size>=0x10000)
+        fail(response_corrupted);
+    flag_alias_len = size;
+    respbuf = client->response;
+    start_response(client, response_groupinfo);
+    pack_now(respbuf, &low, sizeof low);
+    pack_now(respbuf, &high, sizeof high);
+    pack_now(respbuf, &count, sizeof count);
+    pack_now(respbuf, &flag_alias_len, sizeof flag_alias_len);
+    pack_now(respbuf, flag_alias, flag_alias_len);
+    finish_response(client);
+    resetclear(stmt);
+    stmt = NULL;
+    return;
+
+    failhandling_stmt;
+}
+
+static void do_delete_group(
+    client_t *client)
+{
+    buffer_t *reqbuf;
+    void *groupname;
+    uint16_t groupname_len;
+    failvar_stmt;
+
+    reqbuf = client->request;
+    if (!unpack_now(reqbuf, &groupname_len, sizeof groupname_len))
+        fail(response_bad_request);
+    groupname = unpack_later(reqbuf, groupname_len);
+    if (!groupname)
+        fail(response_bad_request);
+    if (!finish_request(client))
+        fail(response_bad_request);
+
+    stmt = sql_main.set_group_deleted;
+    sqlite3_bind_blob(stmt, 1, groupname, groupname_len, SQLITE_STATIC);
+    status = sqlite3_step(stmt);
+    if (status!=SQLITE_DONE)
+        fail_stmt();
+    if (sqlite3_changes(connection)<=0)
+        fail(response_no_group);
+    resetclear(stmt);
+    stmt = NULL;
+    simple_response(client, response_ok);
+    commit_transaction();
+    return;
+
+    failhandling_stmt;
+}
+
+static void do_list_groups(
+    client_t *client)
+{
+    buffer_t *reqbuf, *respbuf;
+    uint32_t space;
+    int64_t groupid;
+    size_t off_code, off_id, off_count;
+    uint32_t groupcount;
+    failvar_stmt;
+
+    reqbuf = client->request;
+    if (!unpack_now(reqbuf, &space, sizeof space))
+        fail(response_bad_request);
+    if (!unpack_now(reqbuf, &groupid, sizeof groupid))
+        fail(response_bad_request);
+    if (!finish_request(client))
+        fail(response_bad_request);
+    if (space>=0x100000)
+        fail(response_bad_request);
+
+    respbuf = client->response;
+    start_response(client, response_grouplist);
+    off_code = respbuf->left-1;
+    off_id = pack_later(respbuf, sizeof groupid);
+    off_count = pack_later(respbuf, sizeof groupcount);
+    groupcount = 0;
+
+    stmt = sql_main.list_groups;
+    sqlite3_bind_int64(stmt, 1, groupid);
+    for (;;) {
+        size_t saveleft = respbuf->left;
+        size_t size;
+        char const *groupname;
+        uint16_t groupname_len;
+        uint64_t low, high, count;
+        char const *flag_alias;
+        uint16_t flag_alias_len;
+
+        status = sqlite3_step(stmt);
+        switch (status) {
+        case SQLITE_ROW:
+            break;
+        case SQLITE_DONE:
+            respbuf->data[off_code] = response_grouplist_done;
+            goto done;
+        default:
+            if (groupcount>0) {
+                goto done;
+            } else {
+                fail_stmt();
+            }
+        }
+
+        groupname = sqlite3_column_blob(stmt, 1);
+        size = sqlite3_column_bytes(stmt, 1);
+        if (!groupname || size>=0x10000)
+            goto corrupted;
+        groupname_len = size;
+        if (pack_now(respbuf, &groupname_len, sizeof groupname_len)>space)
+            goto flush;
+        if (pack_now(respbuf, groupname, groupname_len)>space)
+            goto flush;
+
+        low = sqlite3_column_int64(stmt, 2);
+        if (pack_now(respbuf, &low, sizeof low)>space)
+            goto flush;
+
+        high = sqlite3_column_int64(stmt, 3);
+        if (pack_now(respbuf, &high, sizeof high)>space)
+            goto flush;
+
+        count = sqlite3_column_int64(stmt, 4);
+        if (pack_now(respbuf, &count, sizeof count)>space)
+            goto flush;
+
+        flag_alias = sqlite3_column_blob(stmt, 5);
+        size = sqlite3_column_bytes(stmt, 5);
+        if (!flag_alias || size>=0x10000)
+            goto corrupted;
+        flag_alias_len = size;
+
+        if (pack_now(respbuf, &flag_alias_len, sizeof flag_alias_len)>space)
+            goto flush;
+        if (pack_now(respbuf, flag_alias, flag_alias_len)>space)
+            goto flush;
+
+        groupid = sqlite3_column_int64(stmt, 0);
+        groupcount++;
+        continue;
+
+    corrupted:
+        if (groupcount<=0)
+            fail(response_corrupted);
+    flush:
+        respbuf->left = saveleft;
+        break;
+    }
+
+done:
+    resetclear(stmt);
+    stmt = NULL;
+    memcpy(respbuf->data+off_id, &groupid, sizeof groupid);
+    memcpy(respbuf->data+off_count, &groupcount, sizeof groupcount);
+    finish_response(client);
+    return;
+
+    failhandling_stmt;
+}
+
+static void do_add_article(
+    client_t *client)
+{
+    buffer_t *reqbuf;
+    void *groupname;
+    uint16_t groupname_len;
+    uint64_t artnum;
+    int64_t arrived;
+    int64_t expires;
+    TOKEN token;
+    uint8_t *overview;
+    uint32_t overview_len;
+    int64_t groupid;
+    uint64_t low;
+    failvar_stmt_savepoint;
+
+    reqbuf = client->request;
+    if (!unpack_now(reqbuf, &groupname_len, sizeof groupname_len))
+        fail(response_bad_request);
+    groupname = unpack_later(reqbuf, groupname_len);
+    if (!groupname)
+        fail(response_bad_request);
+    if (!unpack_now(reqbuf, &artnum, sizeof artnum))
+        fail(response_bad_request);
+    if (!unpack_now(reqbuf, &arrived, sizeof arrived))
+        fail(response_bad_request);
+    if (!unpack_now(reqbuf, &expires, sizeof expires))
+        fail(response_bad_request);
+    if (!unpack_now(reqbuf, &token, sizeof token))
+        fail(response_bad_request);
+    if (!unpack_now(reqbuf, &overview_len, sizeof overview_len))
+        fail(response_bad_request);
+    overview = unpack_later(reqbuf, overview_len);
+    if (!overview)
+        fail(response_bad_request);
+    if (!finish_request(client))
+        fail(response_bad_request);
+
+    begin_transaction();
+
+    stmt = sql_main.lookup_groupinfo;
+    sqlite3_bind_blob(stmt, 1, groupname, groupname_len, SQLITE_STATIC);
+    status = sqlite3_step(stmt);
+    switch (status) {
+    case SQLITE_ROW:
+        break;
+    case SQLITE_DONE:
+        fail(response_no_group);
+    default:
+        fail_stmt();
+    }
+    groupid = sqlite3_column_int64(stmt, 0);
+    low = sqlite3_column_int64(stmt, 1);
+    if (client->cutofflow && artnum<low)
+        fail(response_old_article);
+    resetclear(stmt);
+    stmt = NULL;
+
+#ifdef HAVE_ZLIB
+    /*
+     * How to be excessively clever and make the corner cases
+     * work for you instead of against you, part 1.
+     * a) deflation.avail_out is set to the uncompressed size.
+     * b) The uncompressed size is stored first,
+     *    consuming some of the deflation buffer.
+     * c) When deflate returns Z_STREAM_END, we know that everything
+     *    went well _and_ that compression saved at least one byte,
+     *    overhead included.
+     */
+    if (use_compression && overview_len>5) {
+        buffer_resize(flate, overview_len);
+        deflation.next_out = (uint8_t *)flate->data;
+        deflation.avail_out = overview_len;
+        pack_length(&deflation, overview_len);
+        deflation.next_in = overview;
+        deflation.avail_in = overview_len;
+#ifdef USE_DICTIONARY
+        status = deflateSetDictionary(
+            &deflation,
+            (uint8_t *)dictionary,
+            make_dict(groupname, groupname_len, artnum));
+        if (status==Z_OK)
+#endif
+            status = deflate(&deflation, Z_FINISH);
+        flate->left = (char *)deflation.next_out-flate->data;
+        if (status==Z_STREAM_END) {
+            overview = (uint8_t *)flate->data;
+            overview_len = flate->left;
+        } else {
+            /* This is safe; it overwrites the last byte of the overview
+               length, which we have already unpacked. */
+            *--overview = 0;
+            overview_len++;
+        }
+        deflation.next_in = NULL;
+        deflation.avail_in = 0;
+        deflateReset(&deflation);
+    }
+#endif
+
+    savepoint();
+    have_savepoint = true;
+
+    stmt = sql_main.add_article;
+    sqlite3_bind_int64(stmt, 1, groupid);
+    sqlite3_bind_int64(stmt, 2, artnum);
+    sqlite3_bind_int64(stmt, 3, arrived);
+    sqlite3_bind_int64(stmt, 4, expires);
+    sqlite3_bind_blob(stmt, 5, &token, sizeof token, SQLITE_STATIC);
+    sqlite3_bind_blob(stmt, 6, overview, overview_len, SQLITE_STATIC);
+    status = sqlite3_step(stmt);
+    switch (status) {
+    case SQLITE_DONE:
+        break;
+    case SQLITE_CONSTRAINT_PRIMARYKEY:
+        fail(response_dup_article);
+    default:
+        fail_stmt();
+    }
+    resetclear(stmt);
+    stmt = NULL;
+
+    stmt = sql_main.update_groupinfo_add;
+    sqlite3_bind_int64(stmt, 1, groupid);
+    sqlite3_bind_int64(stmt, 2, artnum);
+    status = sqlite3_step(stmt);
+    if (status!=SQLITE_DONE)
+        fail_stmt();
+    resetclear(stmt);
+    stmt = NULL;
+
+    release_savepoint();
+    have_savepoint = false;
+    transaction_rowcount++;
+    simple_response(client, response_ok);
+    return;
+
+    failhandling_stmt_savepoint;
+}
+
+static void do_get_artinfo(
+    client_t *client)
+{
+    buffer_t *reqbuf, *respbuf;
+    void *groupname;
+    uint16_t groupname_len;
+    uint64_t artnum;
+    TOKEN const *token;
+    failvar_stmt;
+
+    reqbuf = client->request;
+    if (!unpack_now(reqbuf, &groupname_len, sizeof groupname_len))
+        fail(response_bad_request);
+    groupname = unpack_later(reqbuf, groupname_len);
+    if (!groupname)
+        fail(response_bad_request);
+    if (!unpack_now(reqbuf, &artnum, sizeof artnum))
+        fail(response_bad_request);
+    if (!finish_request(client))
+        fail(response_bad_request);
+
+    stmt = sql_main.get_artinfo;
+    sqlite3_bind_blob(stmt, 1, groupname, groupname_len, SQLITE_STATIC);
+    sqlite3_bind_int64(stmt, 2, artnum);
+    status = sqlite3_step(stmt);
+    switch (status) {
+    case SQLITE_ROW:
+        break;
+    case SQLITE_DONE:
+        fail(response_no_article);
+    default:
+        fail_stmt();
+    }
+    token = sqlite3_column_blob(stmt, 0);
+    if (!token)
+        fail(response_corrupted);
+    if (sqlite3_column_bytes(stmt, 0) != sizeof (TOKEN))
+        fail(response_corrupted);
+    respbuf = client->response;
+    start_response(client, response_artinfo);
+    pack_now(respbuf, token, sizeof (TOKEN));
+    finish_response(client);
+    resetclear(stmt);
+    stmt = NULL;
+    return;
+
+    failhandling_stmt;
+}
+
+static void do_delete_article(
+    client_t *client)
+{
+    buffer_t *reqbuf;
+    void *groupname;
+    uint16_t groupname_len;
+    uint64_t artnum;
+    int64_t groupid;
+    uint64_t low;
+    failvar_stmt_savepoint;
+
+    reqbuf = client->request;
+    if (!unpack_now(reqbuf, &groupname_len, sizeof groupname_len))
+        fail(response_bad_request);
+    groupname = unpack_later(reqbuf, groupname_len);
+    if (!groupname)
+        fail(response_bad_request);
+    if (!unpack_now(reqbuf, &artnum, sizeof artnum))
+        fail(response_bad_request);
+    if (!finish_request(client))
+        fail(response_bad_request);
+
+    begin_transaction();
+
+    stmt = sql_main.lookup_groupinfo;
+    sqlite3_bind_blob(stmt, 1, groupname, groupname_len, SQLITE_STATIC);
+    status = sqlite3_step(stmt);
+    switch (status) {
+    case SQLITE_ROW:
+        break;
+    case SQLITE_DONE:
+        fail(response_no_group);
+    default:
+        fail_stmt();
+    }
+    groupid = sqlite3_column_int64(stmt, 0);
+    low = sqlite3_column_int64(stmt, 1);
+    resetclear(stmt);
+    stmt = NULL;
+
+    savepoint();
+    have_savepoint = true;
+
+    stmt = sql_main.delete_article;
+    sqlite3_bind_int64(stmt, 1, groupid);
+    sqlite3_bind_int64(stmt, 2, artnum);
+    status = sqlite3_step(stmt);
+    if (status!=SQLITE_DONE)
+        fail_stmt();
+    if (sqlite3_changes(connection)<=0)
+        fail(response_no_article);
+    resetclear(stmt);
+    stmt = NULL;
+
+    if (artnum==low) {
+        stmt = sql_main.update_groupinfo_delete_low;
+    } else {
+        stmt = sql_main.update_groupinfo_delete_middle;
+    }
+    sqlite3_bind_int64(stmt, 1, groupid);
+    sqlite3_bind_int(stmt, 2, 1);
+    status = sqlite3_step(stmt);
+    if (status!=SQLITE_DONE)
+        fail_stmt();
+    resetclear(stmt);
+    stmt = NULL;
+
+    release_savepoint();
+    have_savepoint = false;
+    transaction_rowcount++;
+    simple_response(client, response_ok);
+    return;
+
+    failhandling_stmt_savepoint;
+}
+
+static void do_search_group(
+    client_t *client)
+{
+    buffer_t *reqbuf, *respbuf;
+    uint32_t space;
+    uint8_t flags, cols;
+    void *groupname;
+    uint16_t groupname_len;
+    uint64_t low;
+    uint64_t high;
+    size_t off_count, off_code;
+    uint32_t count;
+    failvar_stmt;
+
+    reqbuf = client->request;
+    if (!unpack_now(reqbuf, &space, sizeof space))
+        fail(response_bad_request);
+    if (!unpack_now(reqbuf, &flags, sizeof flags))
+        fail(response_bad_request);
+    if (!unpack_now(reqbuf, &cols, sizeof cols))
+        fail(response_bad_request);
+    if (!unpack_now(reqbuf, &groupname_len, sizeof groupname_len))
+        fail(response_bad_request);
+    groupname = unpack_later(reqbuf, groupname_len);
+    if (!groupname)
+        fail(response_bad_request);
+    if (!unpack_now(reqbuf, &low, sizeof low))
+        fail(response_bad_request);
+    if (flags & search_flag_high) {
+        if (!unpack_now(reqbuf, &high, sizeof high))
+            fail(response_bad_request);
+    }
+    if (!finish_request(client))
+        fail(response_bad_request);
+    if (space>=0x100000
+            || flags & ~search_flags_all
+            || cols & ~search_cols_all)
+        fail(response_bad_request);
+
+    respbuf = client->response;
+    start_response(client, response_artlist);
+    off_code = respbuf->left-1;
+    pack_now(respbuf, &cols, sizeof cols);
+    off_count = pack_later(respbuf, sizeof count);
+    count = 0;
+
+    if (flags & search_flag_high) {
+        if (cols & search_col_overview) {
+            stmt = sql_main.list_articles_high_overview;
+        } else {
+            stmt = sql_main.list_articles_high;
+        }
+    } else {
+        if (cols & search_col_overview) {
+            stmt = sql_main.list_articles_overview;
+        } else {
+            stmt = sql_main.list_articles;
+        }
+    }
+    sqlite3_bind_blob(stmt, 1, groupname, groupname_len, SQLITE_STATIC);
+    sqlite3_bind_int64(stmt, 2, low);
+    if (flags&search_flag_high)
+        sqlite3_bind_int64(stmt, 3, high);
+    for (;;) {
+        size_t saveleft = respbuf->left;
+        uint64_t artnum;
+        size_t size;
+
+        status = sqlite3_step(stmt);
+        switch (status) {
+        case SQLITE_ROW:
+            break;
+        case SQLITE_DONE:
+            respbuf->data[off_code] = response_artlist_done;
+            goto done;
+        default:
+            if (count>0) {
+                goto done;
+            } else {
+                fail_stmt();
+            }
+        }
+
+        artnum = sqlite3_column_int64(stmt, 0);
+        if (pack_now(respbuf, &artnum, sizeof artnum)>space)
+            goto flush;
+
+        if (cols & search_col_arrived) {
+            int64_t arrived;
+
+            arrived = sqlite3_column_int64(stmt, 1);
+            if (pack_now(respbuf, &arrived, sizeof arrived)>space)
+                goto flush;
+        }
+
+        if (cols & search_col_expires) {
+            int64_t expires;
+
+            expires = sqlite3_column_int64(stmt, 2);
+            if (pack_now(respbuf, &expires, sizeof expires)>space)
+                goto flush;
+        }
+
+        if (cols & search_col_token) {
+            TOKEN const *token;
+
+            token = sqlite3_column_blob(stmt, 3);
+            size = sqlite3_column_bytes(stmt, 3);
+            if (!token || size!=sizeof (TOKEN))
+                goto corrupted;
+            if (pack_now(respbuf, token, sizeof (TOKEN))>space)
+                goto flush;
+        }
+
+        if (cols & search_col_overview) {
+            uint8_t const *overview;
+            uint32_t overview_len;
+
+            overview = sqlite3_column_blob(stmt, 4);
+            size = sqlite3_column_bytes(stmt, 4);
+            if (!overview || size>100000)
+                goto corrupted;
+            overview_len = size;
+#ifdef HAVE_ZLIB
+            /*
+             * How to be excessively clever and make the corner cases
+             * work for you instead of against you, part 2.
+             * a) deflation.avail_out is set to the expected uncompressed size.
+             * b) When inflate returns Z_STREAM_END, we know that everything
+             *    went well _and_ that the uncompressed data isn't larger
+             *    than expected.
+             * c) We still need to check that the uncompressed data isn't
+             *    smaller than expected.
+             */
+            if (use_compression) {
+                uint32_t raw_len;
+
+                inflation.next_in = (uint8_t *)overview;
+                inflation.avail_in = overview_len;
+
+                raw_len = unpack_length(&inflation);
+                if (raw_len>100000)
+                    goto corrupted;
+                if (raw_len>0) {
+                    buffer_resize(flate, raw_len);
+                    inflation.next_out = (uint8_t *)flate->data;
+                    inflation.avail_out = raw_len;
+                    status = inflate(&inflation, Z_FINISH);
+#ifdef USE_DICTIONARY
+                    if (status==Z_NEED_DICT) {
+                        status = inflateSetDictionary(
+                            &inflation,
+                            (uint8_t *)dictionary,
+                            make_dict(groupname, groupname_len, artnum));
+                        if (status==Z_OK)
+                            status = inflate(&inflation, Z_FINISH);
+                    }
+#endif
+                    flate->left = (char *)inflation.next_out-flate->data;
+                    inflation.next_in = NULL;
+                    inflation.avail_in = 0;
+                    inflateReset(&inflation);
+                    if (status!=Z_STREAM_END || inflation.avail_out>0)
+                        goto corrupted;
+                    overview = (uint8_t *)flate->data;
+                    overview_len = flate->left;
+                } else {
+                    overview++;
+                    overview_len--;
+                }
+            }
+#endif
+            if (pack_now(respbuf, &overview_len, sizeof overview_len)>space)
+                goto flush;
+            if (pack_now(respbuf, overview, overview_len)>space)
+                goto flush;
+        }
+
+        count++;
+        continue;
+
+    corrupted:
+        if (count<=0)
+            fail(response_corrupted);
+    flush:
+        respbuf->left = saveleft;
+        break;
+    }
+
+done:
+    resetclear(stmt);
+    stmt = NULL;
+    memcpy(respbuf->data+off_count, &count, sizeof count);
+    finish_response(client);
+    return;
+
+    failhandling_stmt;
+}
+
+static void do_start_expire_group(
+    client_t *client)
+{
+    buffer_t *reqbuf;
+    void *groupname;
+    uint16_t groupname_len;
+    failvar_stmt;
+
+    reqbuf = client->request;
+    if (!unpack_now(reqbuf, &groupname_len, sizeof groupname_len))
+        fail(response_bad_request);
+    groupname = unpack_later(reqbuf, groupname_len);
+    if (!groupname)
+        fail(response_bad_request);
+    if (!finish_request(client))
+        fail(response_bad_request);
+
+    if (!client->expiration_start) {
+        time(&client->expiration_start);
+        client->check_forgotten = true;
+    }
+    stmt = sql_main.start_expire_group;
+    sqlite3_bind_blob(stmt, 1, groupname, groupname_len, SQLITE_STATIC);
+    sqlite3_bind_int64(stmt, 2, client->expiration_start);
+    status = sqlite3_step(stmt);
+    if (status!=SQLITE_DONE)
+        fail_stmt();
+    if (sqlite3_changes(connection)<=0)
+        fail(response_no_group);
+    resetclear(stmt);
+    stmt = NULL;
+    simple_response(client, response_ok);
+    commit_transaction();
+    return;
+
+    failhandling_stmt;
+}
+
+static void do_expire_group(
+    client_t *client)
+{
+    buffer_t *reqbuf;
+    bool hit_low = false;
+    void *groupname;
+    uint16_t groupname_len;
+    uint32_t count, artix;
+    int64_t groupid;
+    uint64_t low;
+    int changes;
+    failvar_stmt_savepoint;
+
+    reqbuf = client->request;
+    if (!unpack_now(reqbuf, &groupname_len, sizeof groupname_len))
+        fail(response_bad_request);
+    groupname = unpack_later(reqbuf, groupname_len);
+    if (!groupname)
+        fail(response_bad_request);
+    if (!unpack_now(reqbuf, &count, sizeof count))
+        fail(response_bad_request);
+
+    if (count<=0) {
+        if (!finish_request(client))
+            fail(response_bad_request);
+        simple_response(client, response_ok);
+        return;
+    }
+
+    begin_transaction();
+
+    stmt = sql_main.lookup_groupinfo;
+    sqlite3_bind_blob(stmt, 1, groupname, groupname_len, SQLITE_STATIC);
+    status = sqlite3_step(stmt);
+    switch (status) {
+    case SQLITE_ROW:
+        break;
+    case SQLITE_DONE:
+        fail(response_no_group);
+    default:
+        fail_stmt();
+    }
+    groupid = sqlite3_column_int64(stmt, 0);
+    low = sqlite3_column_int64(stmt, 1);
+    resetclear(stmt);
+    stmt = NULL;
+
+    savepoint();
+    have_savepoint = true;
+    for (artix=0; artix<count; artix++) {
+        uint64_t artnum;
+
+        if (!unpack_now(reqbuf, &artnum, sizeof artnum))
+            fail(response_bad_request);
+        stmt = sql_main.add_expireart;
+        sqlite3_bind_int64(stmt, 1, artnum);
+        status = sqlite3_step(stmt);
+        if (status!=SQLITE_DONE)
+            fail_stmt();
+        resetclear(stmt);
+        stmt = NULL;
+        if (artnum==low)
+            hit_low = true;
+    }
+    if (!finish_request(client))
+        fail(response_bad_request);
+
+    stmt = sql_main.expire_articles;
+    sqlite3_bind_int64(stmt, 1, groupid);
+    status = sqlite3_step(stmt);
+    if (status!=SQLITE_DONE)
+        fail_stmt();
+    changes = sqlite3_changes(connection);
+    resetclear(stmt);
+    stmt = NULL;
+
+    sqlite3_step(sql_main.clear_expireart);
+    sqlite3_reset(sql_main.clear_expireart);
+
+    if (changes>0) {
+        if (hit_low) {
+            stmt = sql_main.update_groupinfo_delete_low;
+        } else {
+            stmt = sql_main.update_groupinfo_delete_middle;
+        }
+        sqlite3_bind_int64(stmt, 1, groupid);
+        sqlite3_bind_int(stmt, 2, changes);
+        status = sqlite3_step(stmt);
+        if (status!=SQLITE_DONE)
+            fail_stmt();
+        resetclear(stmt);
+        stmt = NULL;
+        transaction_rowcount += changes;
+    }
+
+    release_savepoint();
+    have_savepoint = false;
+    simple_response(client, response_ok);
+    return;
+
+    failhandling_stmt_savepoint;
+}
+
+static void do_finish_expire(
+    client_t *client)
+{
+    int changes;
+    int64_t groupid = 0;
+    failvar_stmt_savepoint;
+
+    if (!finish_request(client))
+        fail(response_bad_request);
+
+    if (!client->expiration_start)
+        fail(response_sequence_error);
+
+    if (client->check_forgotten) {
+        stmt = sql_main.set_forgotten_deleted;
+        sqlite3_bind_int64(stmt, 1, client->expiration_start);
+        status = sqlite3_step(stmt);
+        if (status!=SQLITE_DONE)
+            fail_stmt();
+        changes = sqlite3_changes(connection);
+        resetclear(stmt);
+        stmt = NULL;
+
+        if (changes>0)
+            commit_transaction();
+        client->check_forgotten = false;
+    }
+
+    stmt = sql_main.delete_empty_groups;
+    status = sqlite3_step(stmt);
+    if (status!=SQLITE_DONE)
+        fail_stmt();
+    changes = sqlite3_changes(connection);
+    sqlite3_reset(stmt);
+    stmt = NULL;
+    if (changes>0)
+        commit_transaction();
+
+    begin_transaction();
+
+    stmt = sql_main.get_deleted_groupid;
+    status = sqlite3_step(stmt);
+    switch (status) {
+    case SQLITE_ROW:
+        groupid = sqlite3_column_int64(stmt, 0);
+        break;
+    case SQLITE_DONE:
+        break;
+    default:
+        fail_stmt();
+    }
+    sqlite3_reset(stmt);
+    stmt = NULL;
+
+    if (!groupid) {
+        simple_response(client, response_done);
+        client->expiration_start = 0;
+        return;
+    }
+
+    savepoint();
+    have_savepoint = true;
+
+    stmt = sql_main.fill_expireart;
+    sqlite3_bind_int64(stmt, 1, groupid);
+    sqlite3_bind_int(stmt, 2, transaction_row_limit/2+1);
+    status = sqlite3_step(stmt);
+    if (status!=SQLITE_DONE)
+        fail_stmt();
+    resetclear(stmt);
+    stmt = NULL;
+
+    stmt = sql_main.expire_articles;
+    sqlite3_bind_int64(stmt, 1, groupid);
+    status = sqlite3_step(stmt);
+    if (status!=SQLITE_DONE)
+        fail_stmt();
+    changes = sqlite3_changes(connection);
+    resetclear(stmt);
+    stmt = NULL;
+
+    sqlite3_step(sql_main.clear_expireart);
+    sqlite3_reset(sql_main.clear_expireart);
+
+    transaction_rowcount += changes;
+    release_savepoint();
+    have_savepoint = false;
+    simple_response(client, response_ok);
+    return;
+
+    failhandling_stmt_savepoint;
+}
+
+/*
+ * This needs to stay in sync with the request code enum
+ * in ovsqlite-private.h or things will explode.
+ */
+
+static void (*dispatch[count_request_codes])(
+    client_t *) =
+{
+    do_hello,
+    do_set_cutofflow,
+    do_add_group,
+    do_get_groupinfo,
+    do_delete_group,
+    do_list_groups,
+    do_add_article,
+    do_get_artinfo,
+    do_delete_article,
+    do_search_group,
+    do_start_expire_group,
+    do_expire_group,
+    do_finish_expire
+};
+
+#if defined(EWOULDBLOCK)
+  #if defined(EAGAIN) && EAGAIN!=EWOULDBLOCK
+    #define case_NONBLOCK case EWOULDBLOCK: case EAGAIN:
+  #else
+    #define case_NONBLOCK case EWOULDBLOCK:
+  #endif
+#elif defined(EAGAIN)
+  #define case_NONBLOCK case EAGAIN:
+#else
+  #define case_NONBLOCK
+#endif
+
+static void handle_write(
+    client_t *client)
+{
+    int sock;
+    buffer_t *response;
+    char *data;
+    size_t used, left;
+    ssize_t got;
+
+    sock = client->sock;
+    response = client->response;
+    used = response->used;
+    left = response->left;
+    data = response->data+used;
+    for (;;) {
+        got = write(sock, data, left);
+        if (got>=0 || errno!=EINTR)
+            break;
+    }
+    if (got==-1) {
+        switch (errno) {
+        case_NONBLOCK
+            break;
+        default:
+            del_client(client);
+        }
+        return;
+    }
+    response->used = used+got;
+    response->left = left -= got;
+    if (left>0)
+        return;
+    if (client->flags & client_flag_term) {
+        del_client(client);
+    } else {
+        buffer_set(client->request, NULL, 0);
+        FD_CLR(client->sock, &write_fds);
+        FD_SET(client->sock, &read_fds);
+    }
+}
+
+static void handle_read(
+    client_t *client)
+{
+    unsigned int code;
+
+    for (;;) {
+        bool have_size;
+        size_t left, want;
+        ssize_t got;
+        size_t request_size;
+
+        left = client->request->left;
+        have_size = left>=4;
+        if (have_size) {
+            request_size = *(uint32_t *)(void *)client->request->data;
+            want = request_size-left;
+        } else {
+            want = 5-left;
+        }
+        got = read(client->sock, client->request->data+left, want);
+        if (got==-1) {
+            switch (errno) {
+            case_NONBLOCK
+                break;
+            case EINTR:
+                continue;
+            default:
+                del_client(client);
+            }
+            return;
+        }
+        if (got==0) {
+            del_client(client);
+            return;
+        }
+        client->request->left = left += got;
+        if ((size_t)got<want)
+            return;
+        if (have_size)
+            break;
+        request_size = *(uint32_t *)(void *)client->request->data;
+        if (request_size<5) {
+            simple_response(client, response_bad_request);
+            return;
+        }
+        if (request_size>=0x100000) {
+            simple_response(client, response_oversized);
+            return;
+        }
+        if (left>=request_size)
+            break;
+        buffer_resize(client->request, request_size);
+    }
+    FD_CLR(client->sock, &read_fds);
+    code = start_request(client);
+    if (code>=count_request_codes) {
+        simple_response(client, response_bad_request);
+        return;
+    }
+    if ((code>request_hello) != !(client->flags&client_flag_init)) {
+        simple_response(client, response_wrong_state);
+        return;
+    }
+    (*dispatch[code])(client);
+    return;
+}
+
+static void handle_accept(void)
+{
+#ifdef HAVE_UNIX_DOMAIN_SOCKETS
+    struct sockaddr_un sa;
+#else /* ! HAVE_UNIX_DOMAIN_SOCKETS */
+    struct sockaddr_in sa;
+#endif /* ! HAVE_UNIX_DOMAIN_SOCKETS */
+    socklen_t salen;
+    int sock;
+
+    salen = sizeof sa;
+    sock = accept(listensock, (struct sockaddr *)&sa, &salen);
+    if (sock==-1)
+        return;
+    add_client(sock);
+}
+
+static void mainloop(void)
+{
+    while (!terminating) {
+        fd_set read_fds_out, write_fds_out;
+        int n;
+        struct timeval delta, *nap;
+        client_t *client;
+
+        if (in_transaction) {
+            struct timeval now;
+
+            gettimeofday(&now, NULL);
+            delta = timeval_difference(next_commit, now);
+            if (delta.tv_sec<0 || delta.tv_usec<0
+                    || transaction_rowcount>=transaction_row_limit) {
+                commit_transaction();
+                continue;
+            }
+            nap = δ
+        } else {
+            nap = NULL;
+        }
+        read_fds_out = read_fds;
+        write_fds_out = write_fds;
+        n = select(maxsock+1, &read_fds_out, &write_fds_out, NULL, nap);
+        if (n<=0)
+            continue;
+        if (FD_ISSET(listensock, &read_fds_out)) {
+            n--;
+            handle_accept();
+        }
+        for (client = clients+client_count; n>0 && client>clients; ) {
+            client--;
+            if (FD_ISSET(client->sock, &read_fds_out)) {
+                n--;
+                handle_read(client);
+            } else if (FD_ISSET(client->sock, &write_fds_out)) {
+                n--;
+                handle_write(client);
+            }
+        }
+    }
+    commit_transaction();
+}
+
+static void usage(void)
+{
+    fputs("Usage: ovsqlite-server [ -d ]\n", stderr);
+    exit(1);
+}
+
+int main(
+    int argc,
+    char **argv)
+{
+    bool debug = false;
+
+    setproctitle_init(argc, argv);
+    message_program_name = "ovsqlite-server";
+    for (;;) {
+        int c;
+
+        c = getopt(argc, argv, "d");
+        if (c==-1)
+            break;
+        switch (c) {
+        case 'd':
+            debug = true;
+            break;
+        default:
+            usage();
+        }
+    }
+    if (debug) {
+        message_handlers_warn(1, message_log_stderr);
+        message_handlers_die(1, message_log_stderr);
+    } else {
+        openlog("ovsqlite-server", L_OPENLOG_FLAGS | LOG_PID, LOG_INN_PROG);
+        message_handlers_warn(1, message_log_syslog_err);
+        message_handlers_die(1, message_log_syslog_err);
+    }
+    if (!innconf_read(NULL))
+        exit(1);
+    load_config();
+    if (!debug)
+        daemonize(innconf->pathtmp);
+    catch_signals();
+    make_pidfile();
+    open_db();
+    make_listener();
+    innconf_free(innconf);
+    innconf = NULL;
+    if (setfdlimit(FD_SETSIZE)==-1)
+        syswarn("cannot set file descriptor limit");
+    mainloop();
+    close_sockets();
+    close_db();
+    if (pidfile)
+        unlink(pidfile);
+}
+
+#else /* ! HAVE_SQLITE3 */
+
+int main(void)
+{
+    die("SQLite support not compiled");
+}
+
+#endif /* ! HAVE_SQLITE3 */


Property changes on: trunk/storage/ovsqlite/ovsqlite-server.c
___________________________________________________________________
Added: svn:eol-style
## -0,0 +1 ##
+native
\ No newline at end of property
Added: svn:keywords
## -0,0 +1 ##
+Author Date Id Revision
\ No newline at end of property
Added: storage/ovsqlite/ovsqlite.c
===================================================================
--- storage/ovsqlite/ovsqlite.c	                        (rev 0)
+++ storage/ovsqlite/ovsqlite.c	2020-12-25 09:57:59 UTC (rev 10469)
@@ -0,0 +1,1107 @@
+/*  $Id$
+**
+**  Overview storage method based on SQLite.
+**
+**  Original implementation written by Bo Lindbergh (2020-12-17).
+**  <2bfjdsla52kztwejndzdstsxl9athp at gmail.com>
+*/
+
+#include "ovsqlite.h"
+#include "inn/messages.h"
+#include "ovsqlite-private.h"
+
+#ifdef HAVE_SQLITE3
+
+#include <fcntl.h>
+
+#include "portable/socket.h"
+#ifdef HAVE_UNIX_DOMAIN_SOCKETS
+# include "portable/socket-unix.h"
+#endif
+
+#include "conffile.h"
+#include "inn/fdflag.h"
+#include "inn/innconf.h"
+#include "inn/libinn.h"
+#include "inn/newsuser.h"
+#include "inn/paths.h"
+
+#include "../ovinterface.h"
+
+#define SEARCHSPACE 0x20000
+
+typedef struct handle_t {
+    uint8_t buffer[SEARCHSPACE];
+    uint64_t low;
+    uint64_t high;
+    uint32_t count;
+    uint32_t index;
+    char **overview;
+    time_t *arrived;
+    ARTNUM *artnum;
+    TOKEN *token;
+    uint16_t groupname_len;
+    uint8_t cols;
+    bool done;
+    char groupname[1];
+} handle_t;
+
+static int sock = -1;
+static buffer_t *request;
+static buffer_t *response;
+
+#ifndef HAVE_UNIX_DOMAIN_SOCKETS
+static ovsqlite_port port;
+#endif
+
+static bool server_connect(void)
+{
+    char *path;
+    int ret;
+
+#ifdef HAVE_UNIX_DOMAIN_SOCKETS
+
+    struct sockaddr_un sa;
+
+    sock = socket(PF_UNIX, SOCK_STREAM, 0);
+    if (sock==-1) {
+        syswarn("ovsqlite: socket");
+        return false;
+    }
+    memset(&sa, 0, sizeof sa);
+    sa.sun_family = AF_UNIX;
+    path = concatpath(innconf->pathrun, OVSQLITE_SERVER_SOCKET);
+    strlcpy(sa.sun_path, path, sizeof(sa.sun_path));
+    free(path);
+
+    ret = connect(sock, (struct sockaddr *)&sa, SUN_LEN(&sa));
+
+#else /* ! HAVE_UNIX_DOMAIN_SOCKETS */
+
+    struct sockaddr_in sa;
+    int fd;
+    ssize_t got;
+
+    path = concatpath(innconf->pathrun, OVSQLITE_SERVER_PORT);
+    fd = open(path, O_RDONLY);
+    if (fd==-1) {
+        syswarn("ovsqlite: cannot open port file %s", path);
+        return false;
+    }
+    free(path);
+    got = read(fd, &port, sizeof port);
+    if (got==-1) {
+        syswarn("ovsqlite: cannot read port file");
+        close(fd);
+        return false;
+    }
+    close(fd);
+    if (got<sizeof port) {
+        warn("ovsqlite: unexpected EOF while reading port file");
+        return false;
+    }
+    sock = socket(PF_INET, SOCK_STREAM, 0);
+    if (sock==-1) {
+        syswarn("ovsqlite: socket");
+        return false;
+    }
+    memset(&sa, 0, sizeof sa);
+    sa.sin_family = AF_INET;
+    sa.sin_port = port.port;
+    sa.sin_addr.s_addr = htonl(0x7f000001UL);
+    ret = connect(sock, (struct sockaddr *)&sa, sizeof sa);
+#endif /* ! HAVE_UNIX_DOMAIN_SOCKETS */
+
+    if (ret==-1) {
+        syswarn("ovsqlite: connect");
+        close(sock);
+        sock = -1;
+        return false;
+    }
+
+    request = buffer_new();
+    buffer_resize(request, 0x400);
+    response = buffer_new();
+    buffer_resize(response, 0x400);
+
+    return true;
+}
+
+static void start_request(
+    unsigned int code)
+{
+    uint8_t code_r;
+
+    buffer_set(request, NULL, 0);
+    code_r = code;
+    pack_later(request, 4);
+    pack_now(request, &code_r, sizeof code_r);
+}
+
+static void finish_request(void)
+{
+    *(uint32_t *)(void *)request->data = request->left;
+}
+
+static unsigned int start_response(void)
+{
+    uint8_t code;
+
+    unpack_later(response, 4);
+    unpack_now(response, &code, sizeof code);
+    return code;
+}
+
+static bool finish_response(void)
+{
+    return response->left==0;
+}
+
+static bool write_request(void)
+{
+    char *data;
+    size_t left;
+
+    data = request->data+request->used;
+    left = request->left;
+    while (left>0) {
+        ssize_t got;
+
+        got = write(sock, request->data, request->left);
+        if (got==-1) {
+            if (errno==EINTR)
+                continue;
+            syswarn("ovsqlite: cannot write request");
+            close(sock);
+            sock = -1;
+            return false;
+        }
+        data += got;
+        request->used += got;
+        request->left = left -= got;
+    }
+    return true;
+}
+
+static bool read_response(void)
+{
+    char *data;
+    size_t size, response_size;
+
+    buffer_set(response, NULL, 0);
+    data = response->data;
+    size = 0;
+    response_size = 0;
+    for (;;) {
+        size_t wanted;
+        ssize_t got;
+
+        if (response_size) {
+            wanted = response_size-size;
+        } else {
+            wanted = 5-size;
+        }
+        got = read(sock, data, wanted);
+        if (got==-1) {
+            if (errno==EINTR)
+                continue;
+            syswarn("ovsqlite: cannot read response");
+            close(sock);
+            sock = -1;
+            return false;
+        }
+        if (got==0) {
+            warn("ovsqlite: unexpected EOF while reading response");
+            close(sock);
+            sock = -1;
+            return false;
+        }
+        response->left = size += got;
+        data += got;
+        if ((size_t)got==wanted) {
+            if (response_size) {
+                break;
+            } else {
+                response_size = *(uint32_t *)(void *)response->data;
+                if (response_size<5 || response_size>0x100000) {
+                    warn("ovsqlite: invalid response size");
+                    close(sock);
+                    sock = -1;
+                    return false;
+                }
+                if (size>=response_size)
+                    break;
+                buffer_resize(response, response_size);
+                data = response->data+size;
+            }
+        }
+    }
+    return true;
+}
+
+static bool server_handshake(
+    uint32_t mode)
+{
+    uint32_t version;
+    unsigned int code;
+
+    version = OVSQLITE_PROTOCOL_VERSION;
+    start_request(request_hello);
+    pack_now(request, &version, sizeof version);
+    pack_now(request, &mode, sizeof mode);
+#ifndef HAVE_UNIX_DOMAIN_SOCKETS
+    pack_now(request, port.cookie, OVSQLITE_COOKIE_LENGTH);
+#endif /* ! HAVE_UNIX_DOMAIN_SOCKETS */
+    finish_request();
+    if (!write_request())
+        return false;
+
+    if (!read_response())
+        return false;
+    code = start_response();
+    if (code!=response_ok) {
+        close(sock);
+        sock = -1;
+        warn("ovsqlite: server handshake failed (%u)", code);
+        return false;
+    }
+    if (!finish_response()) {
+        close(sock);
+        sock = -1;
+        warn("ovsqlite: protocol failure");
+        return false;
+    }
+    return true;
+}
+
+bool ovsqlite_open(
+    int mode)
+{
+    if (sock!=-1) {
+        warn("ovsqlite_open called more than once");
+        return false;
+    }
+    if (!server_connect())
+        return false;
+    if (!server_handshake(mode))
+        return false;
+    return true;
+}
+
+bool ovsqlite_groupstats(
+    const char *group,
+    int *low,
+    int *high,
+    int *count,
+    int *flag)
+{
+    uint16_t groupname_len;
+    unsigned int code;
+    uint64_t r_low;
+    uint64_t r_high;
+    uint64_t r_count;
+    uint16_t flag_alias_len;
+    uint8_t *flag_alias;
+
+    if (sock==-1) {
+        warn("ovsqlite: not connected to server");
+        return false;
+    }
+    groupname_len = strlen(group);
+    start_request(request_get_groupinfo);
+    pack_now(request, &groupname_len, sizeof groupname_len);
+    pack_now(request, group, groupname_len);
+    finish_request();
+    if (!write_request())
+        return false;
+
+    if (!read_response())
+        return false;
+    code = start_response();
+    if (code!=response_groupinfo)
+        return false;
+    if (!unpack_now(response, &r_low, sizeof r_low))
+        return false;
+    if (!unpack_now(response, &r_high, sizeof r_high))
+        return false;
+    if (!unpack_now(response, &r_count, sizeof r_count))
+        return false;
+    if (!unpack_now(response, &flag_alias_len, sizeof flag_alias_len))
+        return false;
+    flag_alias = unpack_later(response, flag_alias_len);
+    if (!flag_alias)
+        return false;
+    if (!finish_response())
+        return false;
+    if (low)
+        *low = r_low;
+    if (high)
+        *high = r_high;
+    if (count)
+        *count = r_count;
+    if (flag)
+        *flag = *flag_alias;
+    return true;
+}
+
+bool ovsqlite_groupadd(
+    const char *group,
+    ARTNUM low,
+    ARTNUM high,
+    char *flag)
+{
+    uint16_t groupname_len;
+    uint16_t flag_alias_len;
+    uint64_t r_low;
+    uint64_t r_high;
+    unsigned int code;
+
+    if (sock==-1) {
+        warn("ovsqlite: not connected to server");
+        return false;
+    }
+    groupname_len = strlen(group);
+    r_low = low;
+    r_high = high;
+    flag_alias_len = strcspn(flag, "\n");
+    start_request(request_add_group);
+    pack_now(request, &groupname_len, sizeof groupname_len);
+    pack_now(request, group, groupname_len);
+    pack_now(request, &r_low, sizeof r_low);
+    pack_now(request, &r_high, sizeof r_high);
+    pack_now(request, &flag_alias_len, sizeof flag_alias_len);
+    pack_now(request, flag, flag_alias_len);
+    finish_request();
+    if (!write_request())
+        return false;
+
+    if (!read_response())
+        return false;
+    code = start_response();
+    if (code!=response_ok)
+        return false;
+    if (!finish_response())
+        return false;
+    return true;
+}
+
+bool ovsqlite_groupdel(
+    const char *group)
+{
+    uint16_t groupname_len;
+    unsigned int code;
+
+    if (sock==-1) {
+        warn("ovsqlite: not connected to server");
+        return false;
+    }
+    groupname_len = strlen(group);
+    start_request(request_delete_group);
+    pack_now(request, &groupname_len, sizeof groupname_len);
+    pack_now(request, group, groupname_len);
+    finish_request();
+    if (!write_request())
+        return false;
+
+    if (!read_response())
+        return false;
+    code = start_response();
+    if (code!=response_ok)
+        return false;
+    if (!finish_response())
+        return false;
+    return true;
+}
+
+bool ovsqlite_add(
+    const char *group,
+    ARTNUM artnum,
+    TOKEN token,
+    char *data,
+    int len,
+    time_t arrived,
+    time_t expires)
+{
+    uint16_t groupname_len;
+    uint64_t r_artnum;
+    uint32_t overview_len;
+    uint64_t r_arrived;
+    uint64_t r_expires;
+    unsigned int code;
+
+    if (sock==-1) {
+        warn("ovsqlite: not connected to server");
+        return false;
+    }
+    groupname_len = strlen(group);
+    r_artnum = artnum;
+    overview_len = len;
+    r_arrived = arrived;
+    r_expires = expires;
+    start_request(request_add_article);
+    pack_now(request, &groupname_len, sizeof groupname_len);
+    pack_now(request, group, groupname_len);
+    pack_now(request, &r_artnum, sizeof r_artnum);
+    pack_now(request, &r_arrived, sizeof r_arrived);
+    pack_now(request, &r_expires, sizeof r_expires);
+    pack_now(request, &token, sizeof token);
+    pack_now(request, &overview_len, sizeof overview_len);
+    pack_now(request, data, overview_len);
+    finish_request();
+    if (!write_request())
+        return false;
+
+    if (!read_response())
+        return false;
+    code = start_response();
+    if (!finish_response())
+        return false;
+    switch (code) {
+    case response_ok:
+    case response_no_group:
+        /* Handle unknown newsgroups as a success.
+         * For instance for crossposts to newsgroups not present or no longer
+         * present in active. */
+        return true;
+    default:
+        return false;
+    }
+}
+
+bool ovsqlite_cancel(
+    const char *group,
+    ARTNUM artnum)
+{
+    uint16_t groupname_len;
+    uint64_t r_artnum;
+    unsigned int code;
+
+    if (sock==-1) {
+        warn("ovsqlite: not connected to server");
+        return false;
+    }
+    groupname_len = strlen(group);
+    r_artnum = artnum;
+    start_request(request_delete_article);
+    pack_now(request, &groupname_len, sizeof groupname_len);
+    pack_now(request, group, groupname_len);
+    pack_now(request, &r_artnum, sizeof r_artnum);
+    finish_request();
+    if (!write_request())
+        return false;
+
+    if (!read_response())
+        return false;
+    code = start_response();
+    if (code!=response_ok)
+        return false;
+    if (!finish_response())
+        return false;
+    return true;
+}
+
+void *ovsqlite_opensearch(
+    const char *group,
+    int low,
+    int high)
+{
+    handle_t *rh;
+    uint16_t groupname_len;
+
+    if (sock==-1) {
+        warn("ovsqlite: not connected to server");
+        return NULL;
+    }
+    groupname_len = strlen(group);
+    rh = xmalloc(offsetof(handle_t, groupname)+groupname_len);
+    rh->low = low;
+    rh->high = high;
+    rh->count = 0;
+    rh->index = 0;
+    rh->groupname_len = groupname_len;
+    rh->cols = 0;
+    rh->done = false;
+    memcpy(rh->groupname, group, groupname_len);
+    return rh;
+}
+
+static bool fill_search_buffer(
+    handle_t *rh)
+{
+    unsigned int cols;
+    uint32_t space;
+    uint8_t flags;
+    unsigned int code;
+    uint32_t count, ix;
+    uint8_t *store;
+    size_t wiresize, storesize, storespace;
+    uint8_t resp_cols;
+
+    rh->count = 0;
+    rh->index = 0;
+    storespace = SEARCHSPACE;
+    wiresize = 8;
+    storesize = sizeof (ARTNUM);
+    cols = rh->cols;
+    if (cols & search_col_arrived) {
+        wiresize += 8;
+        storesize += sizeof (time_t);
+    }
+    if (cols & search_col_token) {
+        wiresize += sizeof (TOKEN);
+        storesize += sizeof (TOKEN);
+    }
+    if (cols & search_col_overview) {
+        wiresize += 4;
+        storesize += sizeof (char *);
+        storespace -= sizeof (char *);
+    }
+    space = storespace/storesize*wiresize+10;
+    flags = search_flag_high;
+
+    start_request(request_search_group);
+    pack_now(request, &space, sizeof space);
+    pack_now(request, &flags, sizeof flags);
+    pack_now(request, &rh->cols, sizeof rh->cols);
+    pack_now(request, &rh->groupname_len, sizeof rh->groupname_len);
+    pack_now(request, rh->groupname, rh->groupname_len);
+    pack_now(request, &rh->low, sizeof rh->low);
+    pack_now(request, &rh->high, sizeof rh->high);
+    finish_request();
+    if (!write_request())
+        return false;
+
+    if (!read_response())
+        return false;
+    code = start_response();
+    switch (code) {
+    case response_artlist:
+        break;
+    case response_artlist_done:
+        rh->done = true;
+        break;
+    default:
+        return false;
+    }
+    if (!unpack_now(response, &resp_cols, sizeof resp_cols))
+        return false;
+    if (resp_cols!=cols)
+        return false;
+    if (!unpack_now(response, &count, sizeof count))
+        return false;
+    store = rh->buffer;
+    rh->overview = (char **)(void *)store;
+    if (cols & search_col_overview)
+        store += (count+1)*sizeof (char **);
+    rh->arrived = (time_t *)(void *)store;
+    if (cols & search_col_arrived)
+        store += count*sizeof (time_t);
+    rh->artnum = (ARTNUM *)(void *)store;
+    store += count*sizeof (ARTNUM);
+    rh->token = (TOKEN *)store;
+    if (cols & search_col_token)
+        store += count*sizeof (TOKEN);
+    if (store > rh->buffer+SEARCHSPACE) {
+        warn("ovsqlite: server returned excessive result count");
+        return false;
+    }
+    for (ix=0; ix<count; ix++) {
+        uint64_t artnum;
+
+        if (!unpack_now(response, &artnum, sizeof artnum))
+            return false;
+        rh->artnum[ix] = artnum;
+        if (cols & search_col_arrived) {
+            uint64_t arrived;
+
+            if (!unpack_now(response, &arrived, sizeof arrived))
+                return false;
+            rh->arrived[ix] = arrived;
+        }
+        if (cols & search_col_token) {
+            if (!unpack_now(response, rh->token+ix, sizeof (TOKEN)))
+                return false;
+        }
+        if (cols & search_col_overview) {
+            uint32_t overview_len;
+
+            if (!unpack_now(response, &overview_len, sizeof overview_len))
+                return false;
+            if (store+overview_len > rh->buffer+SEARCHSPACE)
+                return false;
+            if (!unpack_now(response, store, overview_len))
+                return false;
+            rh->overview[ix] = (char *)store;
+            store += overview_len;
+        }
+    }
+    if (!finish_response())
+        return false;
+    if (cols & search_col_overview)
+        rh->overview[count] = (char *)store;
+    rh->count = count;
+    return true;
+}
+
+bool ovsqlite_search(
+    void *handle,
+    ARTNUM *artnum,
+    char **data,
+    int *len,
+    TOKEN *token,
+    time_t *arrived)
+{
+    handle_t *rh;
+    unsigned int cols;
+    unsigned int ix;
+
+    if (sock==-1) {
+        warn("ovsqlite: not connected to server");
+        return false;
+    }
+    rh = handle;
+    if (!rh)
+        return false;
+    ix = rh->index;
+    if (rh->done && ix>=rh->count)
+        return false;
+    cols = 0;
+    if (arrived)
+        cols |= search_col_arrived;
+    if (token)
+        cols |= search_col_token;
+    if (data || len)
+        cols |= search_col_overview;
+    if (cols & ~rh->cols || ix>=rh->count) {
+        rh->cols = cols;
+        if (!fill_search_buffer(rh))
+            return false;
+        ix = rh->index;
+        if (ix>=rh->count)
+            return false;
+    }
+    if (artnum)
+        *artnum = rh->artnum[ix];
+    if (data)
+        *data = rh->overview[ix];
+    if (len)
+        *len = rh->overview[ix+1]-rh->overview[ix];
+    if (token)
+        *token = rh->token[ix];
+    if (arrived)
+        *arrived = rh->arrived[ix];
+    rh->low = rh->artnum[ix]+1;
+    rh->index = ix+1;
+    return true;
+}
+
+void ovsqlite_closesearch(
+    void *handle)
+{
+    if (sock==-1)
+        warn("ovsqlite: not connected to server");
+    if (!handle)
+        return;
+    free(handle);
+}
+
+bool ovsqlite_getartinfo(
+    const char *group,
+    ARTNUM artnum,
+    TOKEN *token)
+{
+    uint16_t groupname_len;
+    uint64_t r_artnum;
+    unsigned int code;
+
+    if (sock==-1) {
+        warn("ovsqlite: not connected to server");
+        return false;
+    }
+    groupname_len = strlen(group);
+    r_artnum = artnum;
+    start_request(request_get_artinfo);
+    pack_now(request, &groupname_len, sizeof groupname_len);
+    pack_now(request, group, groupname_len);
+    pack_now(request, &r_artnum, sizeof r_artnum);
+    finish_request();
+    if (!write_request())
+        return false;
+
+    if (!read_response())
+        return false;
+    code = start_response();
+    if (code!=response_artinfo)
+        return false;
+    if (!unpack_now(response, token, sizeof (TOKEN)))
+        return false;
+    if (!finish_response())
+        return false;
+    return true;
+}
+
+static bool expire_one(
+    char const *group,
+    int *low,
+    struct history *h)
+{
+    unsigned int code;
+    uint32_t space = SEARCHSPACE;
+    uint16_t groupname_len = strlen(group);
+    uint8_t flags = 0;
+    uint8_t cols;
+    uint64_t r_low = 1;
+    ARTNUM new_low = 0;
+    ARTNUM new_high = 0;
+    bool done = false;
+
+    start_request(request_start_expire_group);
+    pack_now(request, &groupname_len, sizeof groupname_len);
+    pack_now(request, group, groupname_len);
+    finish_request();
+    if (!write_request())
+        return false;
+
+    if (!read_response())
+        return false;
+    code = start_response();
+    switch (code) {
+    case response_ok:
+        break;
+    default:
+        return false;
+    }
+
+    cols = search_col_token | search_col_overview;
+    if (innconf->groupbaseexpiry)
+        cols |= search_col_arrived | search_col_expires;
+
+    do {
+        size_t off_count;
+        uint32_t count, ix;
+        uint32_t delcount;
+        uint8_t resp_cols;
+
+        start_request(request_search_group);
+        pack_now(request, &space, sizeof space);
+        pack_now(request, &flags, sizeof flags);
+        pack_now(request, &cols, sizeof cols);
+        pack_now(request, &groupname_len, sizeof groupname_len);
+        pack_now(request, group, groupname_len);
+        pack_now(request, &r_low, sizeof r_low);
+        finish_request();
+        if (!write_request())
+            return false;
+
+        if (!read_response())
+            return false;
+        code = start_response();
+        switch (code) {
+        case response_artlist:
+            break;
+        case response_artlist_done:
+            done = true;
+            break;
+        default:
+            return false;
+        }
+        if (!unpack_now(response, &resp_cols, sizeof resp_cols))
+            return false;
+        if (resp_cols!=cols)
+            return false;
+        if (!unpack_now(response, &count, sizeof count))
+            return false;
+
+        start_request(request_expire_group);
+        pack_now(request, &groupname_len, sizeof groupname_len);
+        pack_now(request, group, groupname_len);
+        delcount = 0;
+        off_count = pack_later(request, sizeof delcount);
+
+        for (ix=0; ix<count; ix++) {
+            uint64_t artnum;
+            uint64_t arrived;
+            uint64_t expires;
+            TOKEN token;
+            char *overview;
+            uint32_t overview_len;
+            bool delete = false;
+            ARTHANDLE *ah = NULL;
+
+            if (!unpack_now(response, &artnum, sizeof artnum))
+                return false;
+            if (innconf->groupbaseexpiry) {
+                if (!unpack_now(response, &arrived, sizeof arrived))
+                    return false;
+                if (!unpack_now(response, &expires, sizeof expires))
+                    return false;
+            }
+            if (!unpack_now(response, &token, sizeof token))
+                return false;
+            if (!unpack_now(response, &overview_len, sizeof overview_len))
+                return false;
+            overview = unpack_later(response, overview_len);
+            if (!overview)
+                return false;
+
+            if (!SMprobe(EXPENSIVESTAT, &token, NULL) || OVstatall) {
+                ah = SMretrieve(token, RETR_STAT);
+                if (ah) {
+                    SMfreearticle(ah);
+                } else {
+                    delete = true;
+                }
+            } else {
+                delete = !OVhisthasmsgid(h, overview);
+            }
+            if (!delete && innconf->groupbaseexpiry) {
+                delete = OVgroupbasedexpire(
+                    token, group, overview, overview_len, arrived, expires);
+            }
+            if (delete) {
+                pack_now(request, &artnum, sizeof artnum);
+                delcount++;
+            } else {
+                if (!new_low)
+                    new_low = artnum;
+            }
+            new_high = artnum;
+            r_low = artnum+1;
+        }
+        if (!finish_response())
+            return false;
+
+        if (delcount>0) {
+            memcpy(request->data+off_count, &delcount, sizeof delcount);
+            finish_request();
+            if (!write_request())
+                return false;
+
+            if (!read_response())
+                return false;
+            code = start_response();
+            if (code!=response_ok)
+                return false;
+            if (!finish_response())
+                return false;
+        }
+    } while (!done);
+    if (!new_low)
+        new_low = new_high+1;
+    if (low)
+        *low = new_low;
+    return true;
+}
+
+static bool expire_finish(void)
+{
+    bool done = false;
+
+    do {
+        unsigned int code;
+
+        start_request(request_finish_expire);
+        finish_request();
+        if (!write_request())
+            return false;
+
+        if (!read_response())
+            return false;
+        code = start_response();
+        switch (code) {
+        case response_ok:
+            break;
+        case response_done:
+            done = true;
+            break;
+        default:
+            return false;
+        }
+        if (!finish_response())
+            return false;
+    } while (!done);
+    return true;
+}
+
+bool ovsqlite_expiregroup(
+    char const *group,
+    int *low,
+    struct history *h)
+{
+    if (sock==-1) {
+        warn("ovsqlite: not connected to server");
+        return false;
+    }
+    if (group) {
+        return expire_one(group, low, h);
+    } else {
+        return expire_finish();
+    }
+}
+
+static bool set_cutofflow(
+    uint8_t cutofflow)
+{
+    unsigned int code;
+
+    start_request(request_set_cutofflow);
+    pack_now(request, &cutofflow, sizeof cutofflow);
+    finish_request();
+    if (!write_request())
+        return false;
+
+    if (!read_response())
+        return false;
+    code = start_response();
+    if (code!=response_ok)
+        return false;
+    if (!finish_response())
+        return false;
+    return true;
+}
+
+bool ovsqlite_ctl(
+    OVCTLTYPE type,
+    void *val)
+{
+    if (sock==-1) {
+        warn("ovsqlite: not connected to server");
+        return false;
+    }
+    switch (type) {
+    case OVSPACE:
+        *(int *)val = -1;
+        return true;
+    case OVSORT:
+        *(OVSORTTYPE *)val = OVNEWSGROUP;
+        return true;
+    case OVCUTOFFLOW:
+        return set_cutofflow(*(bool *)val);
+    case OVSTATICSEARCH:
+        *(int *)val = true;
+        return true;
+    case OVCACHEKEEP:
+    case OVCACHEFREE:
+        *(bool *)val = false;
+        return true;
+    default:
+        return false;
+    }
+}
+
+void ovsqlite_close(void)
+{
+    if (sock==-1) {
+        warn("ovsqlite: not connected to server");
+        return;
+    }
+    close(sock);
+    sock = -1;
+}
+
+#else /* ! HAVE_SQLITE3 */
+
+bool ovsqlite_open(
+    int mode UNUSED)
+{
+    warn("ovsqlite: SQLite support not enabled");
+    return false;
+}
+
+bool ovsqlite_groupstats(
+    const char *group UNUSED,
+    int *lo UNUSED,
+    int *hi UNUSED,
+    int *count UNUSED,
+    int *flag UNUSED)
+{
+    return false;
+}
+
+bool ovsqlite_groupadd(
+    const char *group UNUSED,
+    ARTNUM lo UNUSED,
+    ARTNUM hi UNUSED,
+    char *flag UNUSED)
+{
+    return false;
+}
+
+bool ovsqlite_groupdel(
+    const char *group UNUSED)
+{
+    return false;
+}
+
+bool ovsqlite_add(
+    const char *group UNUSED,
+    ARTNUM artnum UNUSED,
+    TOKEN token UNUSED,
+    char *data UNUSED,
+    int len UNUSED,
+    time_t arrived UNUSED,
+    time_t expires UNUSED)
+{
+    return false;
+}
+
+bool ovsqlite_cancel(
+    const char *group UNUSED,
+    ARTNUM artnum UNUSED)
+{
+    return false;
+}
+
+void *ovsqlite_opensearch(
+    const char *group UNUSED,
+    int low UNUSED,
+    int high UNUSED)
+{
+    return NULL;
+}
+
+bool ovsqlite_search(
+    void *handle UNUSED,
+    ARTNUM *artnum UNUSED,
+    char **data UNUSED,
+    int *len UNUSED,
+    TOKEN *token UNUSED,
+    time_t *arrived UNUSED)
+{
+    return false;
+}
+
+void ovsqlite_closesearch(
+    void *handle UNUSED)
+{
+}
+
+bool ovsqlite_getartinfo(
+    const char *group UNUSED,
+    ARTNUM artnum UNUSED,
+    TOKEN *token UNUSED)
+{
+    return false;
+}
+
+bool ovsqlite_expiregroup(
+    const char *group UNUSED,
+    int *lo UNUSED,
+    struct history *h UNUSED)
+{
+    return false;
+}
+
+bool ovsqlite_ctl(
+    OVCTLTYPE type UNUSED,
+    void *val UNUSED)
+{
+    return false;
+}
+
+void ovsqlite_close(void)
+{
+}
+
+#endif /* ! HAVE_SQLITE3 */
+


Property changes on: trunk/storage/ovsqlite/ovsqlite.c
___________________________________________________________________
Added: svn:eol-style
## -0,0 +1 ##
+native
\ No newline at end of property
Added: svn:keywords
## -0,0 +1 ##
+Author Date Id Revision
\ No newline at end of property
Added: storage/ovsqlite/ovsqlite.h
===================================================================
--- storage/ovsqlite/ovsqlite.h	                        (rev 0)
+++ storage/ovsqlite/ovsqlite.h	2020-12-25 09:57:59 UTC (rev 10469)
@@ -0,0 +1,69 @@
+/*  $Id$
+*/
+
+#ifndef OVSQLITE_H
+#define OVSQLITE_H 1
+
+#include "config.h"
+
+#include "inn/storage.h"
+#include "inn/ov.h"
+
+BEGIN_DECLS
+
+bool ovsqlite_open(
+    int mode);
+bool ovsqlite_groupstats(
+    const char *group,
+    int *lo,
+    int *hi,
+    int *count,
+    int *flag);
+bool ovsqlite_groupadd(
+    const char *group,
+    ARTNUM lo,
+    ARTNUM hi,
+    char *flag);
+bool ovsqlite_groupdel(
+    const char *group);
+bool ovsqlite_add(
+    const char *group,
+    ARTNUM artnum,
+    TOKEN token,
+    char *data,
+    int len,
+    time_t arrived,
+    time_t expires);
+bool ovsqlite_cancel(
+    const char *group,
+    ARTNUM artnum);
+void *ovsqlite_opensearch(
+    const char *group,
+    int low,
+    int high);
+bool ovsqlite_search(
+    void *handle,
+    ARTNUM *artnum,
+    char **data,
+    int *len,
+    TOKEN *token,
+    time_t *arrived);
+void ovsqlite_closesearch(
+    void *handle);
+bool ovsqlite_getartinfo(
+    const char *group,
+    ARTNUM artnum,
+    TOKEN *token);
+bool ovsqlite_expiregroup(
+    const char *group,
+    int *lo,
+    struct history *h);
+bool ovsqlite_ctl(
+    OVCTLTYPE type,
+    void *val);
+void ovsqlite_close(void);
+
+END_DECLS
+
+#endif /* OVSQLITE_H */
+


Property changes on: trunk/storage/ovsqlite/ovsqlite.h
___________________________________________________________________
Added: svn:eol-style
## -0,0 +1 ##
+native
\ No newline at end of property
Added: svn:keywords
## -0,0 +1 ##
+Author Date Id Revision
\ No newline at end of property
Added: storage/ovsqlite/sql-init.sql
===================================================================
--- storage/ovsqlite/sql-init.sql	                        (rev 0)
+++ storage/ovsqlite/sql-init.sql	2020-12-25 09:57:59 UTC (rev 10469)
@@ -0,0 +1,117 @@
+-- ovsqlite schema version 1
+-- $Id$
+
+
+create table misc (
+    key text
+        primary key,
+    value
+        not null
+) without rowid;
+
+-- The misc table holds various settings.
+-- 
+-- These values are set only on database creation:
+--     * 'version'
+--         The value is an integer specifying the schema version.
+--     * 'compress'
+--         The value is a boolean integer specifying whether or not
+--         overview text is stored with zlib compression.
+--     * 'basedict'
+--         Only present if compress is true.
+--         The value is a blob specifying a common prefix for the
+--         dictionaries used with overview text compression.
+--         It contains some fragments commonly found in overview text
+--         and ends with "\tXref: $pathhost ".  For the exact contents,
+--         see basedict_format in ovsqlite-server.c.
+-- 
+-- Currently, no values that vary over the database lifetime exist.
+
+
+create table groupinfo (
+    groupid integer
+        primary key,
+    low integer
+        not null
+        default 1,
+    high integer
+        not null
+        default 0,
+    "count" integer
+        not null
+        default 0,
+    expired integer
+        not null
+        default 0,
+    deleted integer
+        not null
+        default 0,
+    groupname blob
+        not null,
+    flag_alias blob
+        not null,
+
+    unique (deleted, groupname));
+
+-- Newsgroup names aren't guaranteed to use any particular encoding,
+-- so they have to be stored as blobs rather than text.
+-- 
+-- The "flag_alias" column contains the flag in the first byte
+-- and any alias group name in the remaining bytes.
+-- 
+-- The "expired" column contains the time_t of the last expireover run
+-- that included the group; this is used to detect forgotten groups
+-- at the end of the run.
+-- 
+-- When a group is removed, it is marked as deleted and the actual
+-- deletion is deferred until the next expiration.
+-- The "deleted" column is not a boolean; it is set to an unused positive
+-- value on removal.  This supports repeated removal and re-addition of
+-- the same group without any intervening expiration.
+
+
+create table artinfo (
+    groupid integer
+        references groupinfo (groupid)
+	    on update cascade
+            on delete restrict,
+    artnum integer,
+    arrived integer
+        not null,
+    expires integer
+        not null,
+    token blob
+        not null,
+    overview blob
+        not null,
+
+    primary key (groupid, artnum)
+) without rowid;
+
+-- The "arrived" and "expired" columns contain time_t values.
+-- 
+-- The "token" column contains TOKEN values in raw 18-byte format.
+-- 
+-- The "overview" column contains the complete overview data
+-- including the terminating CRLF.
+-- 
+-- Without compression, this is stored unprocessed.
+-- 
+-- With compression, a variable width integer specifying
+-- the uncompressed data size is followed by the compressed data.
+-- In the rare case when compression wouldn't save any space,
+-- a data size of 0 is followed by the uncompressed data.
+-- 
+-- The size is stored in big-endian order and its width is encoded
+-- in the first byte: for width w, the w-1 most significant bits are ones
+-- and the next bit is a zero, leaving w*7 bits for the value itself.
+-- There are no redundant encodings; width 1 is used for values 0 through 127,
+-- width 2 for values 128 through 16511, and so on.
+-- 
+-- Compression uses a dictionary formed by concatenating the common
+-- prefix (stored in the misc table) with "$groupname:$artnum\r\n".
+
+
+-- .getpagesize
+pragma page_size;
+


Property changes on: trunk/storage/ovsqlite/sql-init.sql
___________________________________________________________________
Added: svn:eol-style
## -0,0 +1 ##
+native
\ No newline at end of property
Added: svn:keywords
## -0,0 +1 ##
+Author Date Id Revision
\ No newline at end of property
Added: storage/ovsqlite/sql-main.sql
===================================================================
--- storage/ovsqlite/sql-main.sql	                        (rev 0)
+++ storage/ovsqlite/sql-main.sql	2020-12-25 09:57:59 UTC (rev 10469)
@@ -0,0 +1,203 @@
+pragma foreign_keys = 1;
+
+pragma journal_mode = 'PERSIST';
+
+pragma busy_timeout = 999999999;
+
+-- .random
+select randomblob(?1);
+
+-- .getmisc
+select value from misc
+        where key=?1;
+
+-- .setmisc
+insert or replace into misc (key, value)
+        values (?1, ?2);
+
+-- .unsetmisc
+delete from misc
+        where key=?1;
+
+-- .begin
+begin immediate transaction;
+
+-- .commit
+commit transaction;
+
+-- .rollback
+rollback transaction;
+
+-- .savepoint
+savepoint article_group;
+
+-- .release_savepoint
+release savepoint article_group;
+
+-- .rollback_savepoint
+rollback to savepoint article_group;
+
+-- .delete_journal
+pragma journal_mode = 'DELETE';
+
+-- .add_group
+insert into groupinfo (groupname, flag_alias, low, high)
+    values(?1, ?2, ?3, ?4);
+
+-- .get_groupinfo
+select low, high, "count", flag_alias from groupinfo
+    where deleted=0
+        and groupname=?1;
+
+-- .update_groupinfo_flag_alias
+update groupinfo
+    set flag_alias = ?2
+    where groupid=?1
+        and flag_alias!=?2;
+
+-- .set_group_deleted
+update groupinfo
+    set deleted = (select max(deleted) from groupinfo)+1
+    where deleted=0
+        and groupname=?1;
+
+-- .lookup_groupinfo
+select groupid, low, high, "count" from groupinfo
+    where deleted=0
+        and groupname=?1;
+
+-- .list_groups
+select groupid, groupname, low, high, "count", flag_alias from groupinfo
+    where deleted=0
+        and groupid>?1
+    order by groupid;
+
+-- .add_article
+insert into artinfo (groupid, artnum, arrived, expires, token, overview)
+    values(?1, ?2, ?3, ?4, ?5, ?6);
+
+-- .update_groupinfo_add
+update groupinfo
+    set low = case when "count" then min(low, ?2) else ?2 end,
+        high = case when "count" then max(high, ?2) else ?2 end,
+        "count" = "count"+1
+    where groupid=?1;
+
+-- .get_artinfo
+select token
+    from groupinfo
+        natural join artinfo
+    where deleted=0
+        and groupname=?1
+        and artnum=?2;
+
+-- .delete_article
+delete from artinfo
+    where groupid=?1
+        and artnum=?2;
+
+-- .update_groupinfo_delete_low
+update groupinfo
+    set low = coalesce(
+            (select min(artnum) from artinfo
+                where artinfo.groupid=groupinfo.groupid),
+            high+1),
+        "count" = "count"-?2
+    where groupid=?1;
+
+-- .update_groupinfo_delete_middle
+update groupinfo
+    set "count" = "count"-?2
+    where groupid=?1;
+
+-- .start_expire_group
+update groupinfo
+    set expired = ?2
+        where deleted=0
+            and groupname=?1;
+
+--
+create temporary table expireart(
+    artnum integer
+        primary key);
+
+-- .add_expireart
+insert or ignore into expireart (artnum)
+    values(?1);
+
+-- .fill_expireart
+insert or ignore into expireart (artnum)
+    select artnum from artinfo
+        where groupid=?1
+        order by artnum
+        limit ?2;
+
+-- .expire_articles
+delete from artinfo
+    where groupid=?1
+        and artnum in expireart;
+
+-- .clear_expireart
+delete from expireart;
+
+-- .set_forgotten_deleted
+update groupinfo
+    set deleted = (select max(deleted) from groupinfo)+1
+    where deleted=0
+        and expired<?1;
+
+-- .delete_empty_groups
+delete from groupinfo
+    where deleted>0
+        and not exists
+            (select * from artinfo
+                where artinfo.groupid=groupinfo.groupid);
+
+-- .get_deleted_groupid
+select groupid from groupinfo
+    where deleted>0
+    order by deleted
+    limit 1;
+
+-- .delete_group
+delete from groupinfo
+    where groupid=?1;
+
+-- .list_articles
+select artnum, arrived, expires, token
+    from groupinfo
+        natural join artinfo
+    where deleted=0
+        and groupname=?1
+        and artnum>=?2
+    order by artnum;
+
+-- .list_articles_overview
+select artnum, arrived, expires, token, overview
+    from groupinfo
+        natural join artinfo
+    where deleted=0
+        and groupname=?1
+        and artnum>=?2
+    order by artnum;
+
+-- .list_articles_high
+select artnum, arrived, expires, token
+    from groupinfo
+        natural join artinfo
+    where deleted=0
+        and groupname=?1
+        and artnum>=?2
+        and artnum<=?3
+    order by artnum;
+
+-- .list_articles_high_overview
+select artnum, arrived, expires, token, overview
+    from groupinfo
+        natural join artinfo
+    where deleted=0
+        and groupname=?1
+        and artnum>=?2
+        and artnum<=?3
+    order by artnum;
+


Property changes on: trunk/storage/ovsqlite/sql-main.sql
___________________________________________________________________
Added: svn:eol-style
## -0,0 +1 ##
+native
\ No newline at end of property
Added: svn:keywords
## -0,0 +1 ##
+Author Date Id Revision
\ No newline at end of property
Added: storage/ovsqlite/sqlite-helper-gen.in
===================================================================
--- storage/ovsqlite/sqlite-helper-gen.in	                        (rev 0)
+++ storage/ovsqlite/sqlite-helper-gen.in	2020-12-25 09:57:59 UTC (rev 10469)
@@ -0,0 +1,209 @@
+#! /usr/bin/perl
+
+# $Id$
+#
+# The sqlite-helper-gen script reads SQL text from its input and splits it
+# into sections using lines starting with a comment ("--") as delimiters.
+# 
+# If the comment is followed by a period (".") and a valid C identifier and
+# nothing else but whitespace, succeeding lines go into the prepared
+# statement section named by the identifier.
+# 
+# If the comment is followed by nothing but whitespace, succeeding lines go
+# into the unnamed initialisation section.
+# 
+# Otherwise, the destination section remains unchanged.
+# 
+# After the entire input is read, sqlite-helper-gen writes two output
+# files: a C header file and a C source file.
+# 
+# The header file contains this:
+#   * a struct type definition with a prepared statement pointer
+#     for each named statement found in the input
+#   * a declaration of a sqlite-helper variable
+# 
+# The source file contains this:
+#   * the definition for the sqlite-helper variable, initialised
+#     with the processed SQL text from the input
+# 
+# This allows application code to do all this with a single call to the
+# sqlite-helper code:
+#   * execute the statements in the initialisation section
+#   * prepare each named statement
+#
+# Original implementation written by Bo Lindbergh (2020-12-17).
+# <2bfjdsla52kztwejndzdstsxl9athp at gmail.com>
+
+use strict;
+use warnings;
+
+sub parse
+{
+    my($path, $state) = @_;
+    my($fh, %sections, @names, $cursect, @cursect);
+    my $donesect=sub
+    {
+        while (@cursect && $cursect[-1] !~ /\S/) {
+            pop(@cursect);
+        }
+        while (@cursect && $cursect[0] !~ /\S/) {
+            shift(@cursect);
+        }
+        push(@{$cursect}, splice(@cursect));
+    };
+
+    open($fh, "<", $path)
+        or die "$path: open: $!\n";
+    binmode($fh);
+    $state->{sections} = \%sections;
+    $state->{names} = \@names;
+    $cursect = \@{$sections{""}};
+    while (defined(my $line = readline($fh))) {
+        $line =~ tr/\0//d;
+        if ($line =~ /^\s*--\s*/g) {
+            if ($line =~ /\G\.\s*([_a-zA-Z][_a-zA-Z0-9]*)\s*$/gc) {
+                $donesect->();
+                if (!exists($sections{$1})) {
+                    push(@names, $1);
+                }
+                $cursect = \@{$sections{$1}};
+            } elsif ($line =~ /\G$/gc) {
+                $donesect->();
+                $cursect = \@{$sections{""}};
+            }
+        } else {
+            push(@cursect, $line);
+        }
+    }
+    $donesect->();
+    close($fh);
+    $state->{preamble} = <<"__PREAMBLE__"
+/*
+ * DO NOT EDIT DIRECTLY.
+ *
+ * This file was generated automatically by sqlite-helper-gen
+ * from $path.
+ *
+ * DO NOT EDIT DIRECTLY.
+ */
+__PREAMBLE__
+        ;
+}
+
+sub generate_head
+{
+    my($path, $state) = @_;
+    my($fh, $names, $name, $preamble, $guard, $stmts);
+
+    open($fh, ">", $path)
+        or die "$path: open: $!\n";
+    binmode($fh);
+    $names = $state->{names};
+    $name = $state->{name};
+    $preamble = $state->{preamble};
+    $guard = "\U$name\E_H";
+    unless (@{$names}) {
+        $names=["unused"];
+    }
+    $stmts = join("\n    ", map("sqlite3_stmt *$_;", @{$names}));
+    print $fh <<"__HEAD__"
+$preamble
+#ifndef $guard
+#define $guard 1
+
+#include "sqlite-helper.h"
+
+#ifdef HAVE_SQLITE3
+
+typedef struct ${name}_t {
+    $stmts
+} ${name}_t;
+
+extern sqlite_helper_t const ${name}_helper;
+
+#endif /* HAVE_SQLITE3 */
+
+#endif /* ! $guard */
+__HEAD__
+        or die "$path: print: $!\n";
+    close($fh)
+        or die "$path: close: $!\n";
+}
+
+my(%escape, $escapevals);
+
+BEGIN {
+    %escape = (
+        "\\"    => "\\\\",
+        "\""    => "\\\"",
+        "\r"    => "\\r",
+        "\n"    => "\\n",
+        "\t"    => "\\t",
+    );
+    $escapevals = join("", values(%escape));
+}
+
+sub generate_body
+{
+    my($path, $state) = @_;
+    my($fh, $sections, $names, $name, $base, $preamble);
+    my($stmt_count, @lines, $text);
+
+    open($fh, ">", $path)
+        or die "$path: open: $!\n";
+    binmode($fh);
+    $sections = $state->{sections};
+    $names = $state->{names};
+    $name = $state->{name};
+    $base = $state->{base};
+    $preamble = $state->{preamble};
+    push(@lines, splice(@{$sections->{""}}));
+    push(@lines, "\0");
+    $stmt_count = @{$names};
+    foreach my $name (@{$names}) {
+        push(@lines, "$name\0");
+        push(@lines, splice(@{$sections->{$name}}));
+        push(@lines, "\0");
+    }
+    foreach my $line (@lines) {
+        $line =~ s/([$escapevals]|[^ -~])/
+            $escape{$1} || sprintf("\\%.3o", ord($1))/ge;
+        $line="\"$line\"";
+    }
+    $text = join("\n    ", at lines);
+    print $fh <<"__BODY__"
+$preamble
+
+#include "$base.h"
+
+#ifdef HAVE_SQLITE3
+
+sqlite_helper_t const ${name}_helper =
+{
+    $stmt_count,
+
+    $text
+};
+
+#endif /* HAVE_SQLITE3 */
+
+__BODY__
+        or die "$path: print: $!\n";
+    close($fh)
+        or die "$path: close: $!\n";
+}
+
+foreach my $src (@ARGV) {
+    my($dir, $base, $name, %state);
+
+    ($dir,$base) = $src =~ m,^((?:.*/)?)([-_a-zA-Z][-_a-zA-Z0-9]*)[^/]*$,
+        or die "$src: Bad input filename\n";
+    $name = $base;
+    $name =~ tr/-/_/;
+    $state{name} = $name;
+    $state{base} = $base;
+    parse($src, \%state);
+    generate_head("$dir$base.h", \%state);
+    generate_body("$dir$base.c", \%state);
+}
+


Property changes on: trunk/storage/ovsqlite/sqlite-helper-gen.in
___________________________________________________________________
Added: svn:eol-style
## -0,0 +1 ##
+native
\ No newline at end of property
Added: svn:keywords
## -0,0 +1 ##
+Author Date Id Revision
\ No newline at end of property
Added: storage/ovsqlite/sqlite-helper.c
===================================================================
--- storage/ovsqlite/sqlite-helper.c	                        (rev 0)
+++ storage/ovsqlite/sqlite-helper.c	2020-12-25 09:57:59 UTC (rev 10469)
@@ -0,0 +1,101 @@
+/*  $Id$
+*/
+
+#include "sqlite-helper.h"
+
+#ifdef HAVE_SQLITE3
+
+#include <string.h>
+#include <stdio.h>
+
+/* sqlite3_prepare_v3() is defined in SQLite 3.20.0 and above. */
+#if SQLITE_VERSION_NUMBER >= 3020000
+# define UNUSED_BEFORE_SQLITE_3_20
+#else
+# define UNUSED_BEFORE_SQLITE_3_20 UNUSED
+# define sqlite3_prepare_v3(db, zSql, nByte, prepFlags, ppStmt, pzTail) \
+         sqlite3_prepare_v2(db, zSql, nByte, ppStmt, pzTail)
+#endif
+
+int sqlite_helper_init(
+    sqlite_helper_t const *helper,
+    sqlite3_stmt **stmts,
+    sqlite3 *connection,
+    unsigned int prepare_flags UNUSED_BEFORE_SQLITE_3_20,
+    char **errmsg)
+{
+    int result;
+    size_t ix, count;
+    char const *text, *textend;
+    char const *name, *nameend;
+
+    text = helper->text;
+    result = sqlite3_exec(connection, text, 0, NULL, errmsg);
+    if (result!=SQLITE_OK)
+        return result;
+    name = strchr(text, 0)+1;
+    count = helper->stmt_count;
+    for (ix=0; ix<count; ix++) {
+        nameend = strchr(name, 0);
+        text = nameend+1;
+        textend = strchr(text, 0);
+        result=sqlite3_prepare_v3(
+            connection,
+            text, textend-text,
+            prepare_flags,
+            stmts+ix,
+            NULL);
+        if (result!=SQLITE_OK)
+            goto oops;
+        name = textend+1;
+    }
+    return SQLITE_OK;
+
+oops:
+    if (errmsg) {
+        size_t namelen;
+        char const *err;
+        char *errcopy;
+
+        namelen = nameend-name;
+        err = sqlite3_errmsg(connection);
+        if (err) {
+            size_t errlen;
+
+            errlen = strlen(err);
+            errcopy = sqlite3_malloc64(namelen+errlen+3);
+            if (errcopy) {
+                memcpy(errcopy, name, namelen);
+                errcopy[namelen]=':';
+                errcopy[namelen+1]=' ';
+                memcpy(errcopy+namelen+2, err, errlen+1);
+            }
+        } else {
+            errcopy = sqlite3_malloc64(namelen+1);
+            if (errcopy)
+                memcpy(errcopy, name, namelen+1);
+        }
+        *errmsg = errcopy;
+    }
+    while (ix-->0) {
+        sqlite3_finalize(stmts[ix]);
+        stmts[ix] = NULL;
+    }
+    return result;
+}
+
+void sqlite_helper_term(
+    sqlite_helper_t const *helper,
+    sqlite3_stmt **stmts)
+{
+    size_t ix, count;
+
+    count = helper->stmt_count;
+    for (ix=0; ix<count; ix++) {
+        sqlite3_finalize(stmts[ix]);
+        stmts[ix] = NULL;
+    }
+}
+
+#endif /* HAVE_SQLITE3 */
+


Property changes on: trunk/storage/ovsqlite/sqlite-helper.c
___________________________________________________________________
Added: svn:eol-style
## -0,0 +1 ##
+native
\ No newline at end of property
Added: svn:keywords
## -0,0 +1 ##
+Author Date Id Revision
\ No newline at end of property
Added: storage/ovsqlite/sqlite-helper.h
===================================================================
--- storage/ovsqlite/sqlite-helper.h	                        (rev 0)
+++ storage/ovsqlite/sqlite-helper.h	2020-12-25 09:57:59 UTC (rev 10469)
@@ -0,0 +1,55 @@
+/*  $Id$
+*/
+
+#ifndef SQLITE_HELPER_H
+#define SQLITE_HELPER_H 1
+
+#include "config.h"
+
+#ifdef HAVE_SQLITE3
+
+#include <stddef.h>
+#include <sqlite3.h>
+
+/* SQLITE_PREPARE_PERSISTENT is defined in SQLite 3.20.0 and above. */
+#ifndef SQLITE_PREPARE_PERSISTENT
+# define SQLITE_PREPARE_PERSISTENT 0x00
+#endif
+
+typedef struct sqlite_helper_t {
+    size_t stmt_count;
+    char const *text;
+} sqlite_helper_t;
+
+BEGIN_DECLS
+
+/*
+ * Execute initialisation statements and prepare named statements.
+ *
+ * Returns a SQLite error code.
+ *
+ * In case of error, the caller is responsible for deallocating
+ * the returned error message with sqlite3_free.
+ */
+
+extern int sqlite_helper_init(
+    sqlite_helper_t const *helper,
+    sqlite3_stmt **stmts,
+    sqlite3 *connection,
+    unsigned int prepare_flags,
+    char **errmsg);
+
+/*
+ * Deallocate all the prepared statements.
+ */
+
+extern void sqlite_helper_term(
+    sqlite_helper_t const *helper,
+    sqlite3_stmt **stmts);
+
+END_DECLS
+
+#endif /* HAVE_SQLITE3 */
+
+#endif /* ! SQLITE_HELPER_H */
+


Property changes on: trunk/storage/ovsqlite/sqlite-helper.h
___________________________________________________________________
Added: svn:eol-style
## -0,0 +1 ##
+native
\ No newline at end of property
Added: svn:keywords
## -0,0 +1 ##
+Author Date Id Revision
\ No newline at end of property
Modified: support/getrra-c-util
===================================================================
--- support/getrra-c-util	2020-12-25 09:51:06 UTC (rev 10468)
+++ support/getrra-c-util	2020-12-25 09:57:59 UTC (rev 10469)
@@ -60,6 +60,37 @@
         sed -i -e '/dnl Provides INN_PROG_CC_WARNINGS_FLAGS/,+6d' \
                -e '/dnl Determine the full set/,$d' \
              ${TEMP}
+      elif [ "$3" = "sqlite3.m4" ]
+      then
+        sed -i -e "43 i \\
+dnl Ensures SQLite v3 meets our minimum version requirement.\\
+AC_DEFUN([_INN_LIB_SQLITE3_SOURCE], [[\\
+#include <sqlite3.h>\\
+\\
+int main(void) {\\
+    return sqlite3_libversion_number() < 3008002;\\
+}\\
+]])\\
+\\
+dnl Checks if SQLite v3 is present.  The single argument, if \"true\", says to\\
+dnl fail if the SQLite library could not be found.\\
+AC_DEFUN([_INN_LIB_SQLITE3_INTERNAL],\\
+[AC_CACHE_CHECK([for a sufficiently recent SQLite],\\
+    [inn_cv_have_sqlite3],\\
+    [INN_LIB_HELPER_PATHS([SQLITE3])\\
+     INN_LIB_SQLITE3_SWITCH\\
+     LIBS=\"-lsqlite3 \$LIBS\"\\
+     AC_RUN_IFELSE([AC_LANG_SOURCE([_INN_LIB_SQLITE3_SOURCE])],\\
+        [inn_cv_have_sqlite3=yes],\\
+        [inn_cv_have_sqlite3=no])\\
+     INN_LIB_SQLITE3_RESTORE])\\
+ AS_IF([test x\"\$inn_cv_have_sqlite3\" = xyes],\\
+    [SQLITE3_LIBS=\"-lsqlite3\"],\\
+    [AS_IF([test x\"\$1\" = xtrue],\\
+        [AC_MSG_ERROR([cannot find usable SQLite v3 library])])])])\\
+" \
+               -e '43,63d' \
+             ${TEMP}
       fi
     elif [ "$2" = "include/inn" ] || [ "$2" = "include/portable" ] \
       || [ "$2" = "include" ] || [ "$2" = "lib" ] || [ "$2" = "tests/lib" ] \
@@ -257,6 +288,7 @@
 download m4/sasl.m4 m4 sasl.m4
 download m4/socket-unix.m4 m4 socket-unix.m4
 download m4/socket.m4 m4 socket.m4
+download m4/sqlite3.m4 m4 sqlite3.m4
 download m4/vamacros.m4 m4 vamacros.m4
 download m4/zlib.m4 m4 zlib.m4
 

Modified: support/mkmanifest
===================================================================
--- support/mkmanifest	2020-12-25 09:51:06 UTC (rev 10468)
+++ support/mkmanifest	2020-12-25 09:57:59 UTC (rev 10469)
@@ -251,6 +251,7 @@
 site/nntpsend.ctl
 site/nocem.ctl
 site/ovdb.conf
+site/ovsqlite.conf
 site/passwd.nntp
 site/readers.conf
 site/send-uucp.cf
@@ -259,8 +260,10 @@
 site/subscriptions
 site/update
 snapshot.log
+storage/buffindexed/buffindexed_d
 storage/buildconfig
-storage/buffindexed/buffindexed_d
+storage/ovsqlite/ovsqlite-server
+storage/ovsqlite/sqlite-helper-gen
 storage/tradindexed/tdx-util
 support/fixconfig
 support/fixscript

Modified: tests/Makefile
===================================================================
--- tests/Makefile	2020-12-25 09:51:06 UTC (rev 10468)
+++ tests/Makefile	2020-12-25 09:57:59 UTC (rev 10469)
@@ -4,7 +4,7 @@
 
 top		= ..
 RUNTESTS_CFLAGS	= -DC_TAP_SOURCE='"$(abs_builddir)/tests"' -DC_TAP_BUILD='"$(abs_builddir)/tests"'
-CFLAGS		= $(GCFLAGS) $(BDB_CPPFLAGS) $(DBM_CPPFLAGS) $(PERL_CPPFLAGS) $(PYTHON_CPPFLAGS) $(SSL_CPPFLAGS) $(SASL_CPPFLAGS) $(KRB5_CPPFLAGS) $(RUNTESTS_CFLAGS) -I.
+CFLAGS		= $(GCFLAGS) $(BDB_CPPFLAGS) $(DBM_CPPFLAGS) $(SQLITE3_CPPFLAGS) $(PERL_CPPFLAGS) $(PYTHON_CPPFLAGS) $(SSL_CPPFLAGS) $(SASL_CPPFLAGS) $(KRB5_CPPFLAGS) $(RUNTESTS_CFLAGS) -I.
 
 ##  On some platforms, linking with libm is needed as the test suite uses
 ##  math functions (currently only lib/confparse-t.c).



More information about the inn-committers mailing list