--- a/roundup/configuration.py	Fri Oct 08 00:37:16 2021 -0400
+++ b/roundup/configuration.py	Thu Apr 21 16:54:17 2022 -0400
@@ -9,7 +9,9 @@
 # used here instead of try/except.
 import sys
 import getopt
-import logging, logging.config
+import errno
+import logging
+import logging.config
 import os
 import re
 import time
@@ -31,8 +33,6 @@
 
 from roundup.exceptions import RoundupException
 
-# XXX i don't think this module needs string translation, does it?
-
 ### Exceptions
 
 
@@ -40,6 +40,11 @@
     pass
 
 
+class ParsingOptionError(ConfigurationError):
+    def __str__(self):
+        return self.args[0]
+
+
 class NoConfigError(ConfigurationError):
 
     """Raised when configuration loading fails
@@ -261,8 +266,17 @@
 
     def load_ini(self, config):
         """Load value from ConfigParser object"""
-        if config.has_option(self.section, self.setting):
-            self.set(config.get(self.section, self.setting))
+        try:
+            if config.has_option(self.section, self.setting):
+                self.set(config.get(self.section, self.setting))
+        except configparser.InterpolationSyntaxError as e:
+            raise ParsingOptionError(
+                _("Error in %(filepath)s with section [%(section)s] at "
+                  "option %(option)s: %(message)s") % {
+                      "filepath": self.config.filepath,
+                      "section": e.section,
+                      "option": e.option,
+                      "message": str(e)})
 
 
 class BooleanOption(Option):
@@ -408,19 +422,91 @@
             return _val
         raise OptionValueError(self, value, self.class_description)
 
+
 class IndexerOption(Option):
     """Valid options for indexer"""
 
-    allowed = ['', 'xapian', 'whoosh', 'native']
+    allowed = ['', 'xapian', 'whoosh', 'native', 'native-fts']
     class_description = "Allowed values: %s" % ', '.join("'%s'" % a
                                                          for a in allowed)
 
+    # FIXME this is the result of running:
+    #    SELECT cfgname FROM pg_ts_config;
+    # on a postgresql 14.1 server.
+    # So the best we can do is hardcode this.
+    valid_langs = [ "simple",
+                    "custom1",
+                    "custom2",
+                    "custom3",
+                    "custom4",
+                    "custom5",
+                    "arabic",
+                    "armenian",
+                    "basque",
+                    "catalan",
+                    "danish",
+                    "dutch",
+                    "english",
+                    "finnish",
+                    "french",
+                    "german",
+                    "greek",
+                    "hindi",
+                    "hungarian",
+                    "indonesian",
+                    "irish",
+                    "italian",
+                    "lithuanian",
+                    "nepali",
+                    "norwegian",
+                    "portuguese",
+                    "romanian",
+                    "russian",
+                    "serbian",
+                    "spanish",
+                    "swedish",
+                    "tamil",
+                    "turkish",
+                    "yiddish" ]
+                                                         
     def str2value(self, value):
         _val = value.lower()
         if _val in self.allowed:
             return _val
         raise OptionValueError(self, value, self.class_description)
 
+    def validate(self, options):
+
+        if self._value in ("", "xapian"):
+            try:
+                import xapian
+            except ImportError:
+                # indexer is probably '' and xapian isn't present
+                # so just return at end of method
+                pass
+            else:
+                try:
+                    lang = options["INDEXER_LANGUAGE"]._value
+                    xapian.Stem(lang)
+                except xapian.InvalidArgumentError:
+                    import textwrap
+                    lang_avail = b2s(xapian.Stem.get_available_languages())
+                    languages = textwrap.fill(_("Valid languages: ") +
+                                              lang_avail, 75,
+                                              subsequent_indent="   ")
+                    raise OptionValueError(options["INDEXER_LANGUAGE"],
+                                            lang, languages)
+
+        if self._value == "native-fts":
+            lang = options["INDEXER_LANGUAGE"]._value
+            if lang not in self.valid_langs:
+                import textwrap
+                languages = textwrap.fill(_("Expected languages: ") +
+                                            " ".join(self.valid_langs), 75,
+                                            subsequent_indent="   ")
+                raise OptionValueError(options["INDEXER_LANGUAGE"],
+                                       lang, languages)
+
 class MailAddressOption(Option):
 
     """Email address
@@ -552,6 +638,62 @@
             return value
 
 
+class SecretOption(Option):
+    """A string not beginning with file:// or a file starting with file://
+
+    Paths may be either absolute or relative to the HOME.
+    Value for option is the first line in the file.
+    It is mean to store secret information in the config file but
+    allow the config file to be stored in version control without
+    storing the secret there.
+
+    """
+
+    class_description = \
+      "A string that starts with 'file://' is interpreted as a file path \n" \
+      "relative to the tracker home. Using 'file:///' defines an absolute \n" \
+      "path. The first line of the file will be used as the value. Any \n" \
+      "string that does not start with 'file://' is used as is. It \n" \
+      "removes any whitespace at the end of the line, so a newline can \n" \
+      "be put in the file.\n"
+
+    def get(self):
+        _val = Option.get(self)
+        if isinstance(_val, str) and _val.startswith('file://'):
+            filepath = _val[7:]
+            if filepath and not os.path.isabs(filepath):
+                filepath = os.path.join(self.config["HOME"], filepath.strip())
+            try:
+                with open(filepath) as f:
+                    _val = f.readline().rstrip()
+            # except FileNotFoundError: py2/py3
+            # compatible version
+            except EnvironmentError as e:
+                if e.errno != errno.ENOENT:
+                    raise
+                else:
+                    raise OptionValueError(self, _val,
+                        "Unable to read value for %s. Error opening "
+                        "%s: %s." % (self.name, e.filename, e.args[1]))
+        return self.str2value(_val)
+
+    def validate(self, options):
+        if self.name == 'MAIL_PASSWORD':
+            if options['MAIL_USERNAME']._value:
+                # MAIL_PASSWORD is an exception. It is mandatory only
+                # if MAIL_USERNAME is set. So check only if username
+                # is set.
+                try:
+                    self.get()
+                except OptionUnsetError:
+                    # provide error message with link to MAIL_USERNAME
+                    raise OptionValueError(options["MAIL_PASSWORD"],
+                                           "not defined",
+                            "Mail username is set, so this must be defined.")
+        else:
+            self.get()
+
+
 class WebUrlOption(Option):
     """URL MUST start with http/https scheme and end with '/'"""
 
@@ -614,6 +756,18 @@
     # everything else taken from NullableOption (inheritance order)
 
 
+class SecretMandatoryOption(MandatoryOption, SecretOption):
+    # use get from SecretOption and rest from MandatoryOption
+    get = SecretOption.get
+    class_description = SecretOption.class_description
+
+
+class SecretNullableOption(NullableOption, SecretOption):
+    # use get from SecretOption and rest from NullableOption
+    get = SecretOption.get
+    class_description = SecretOption.class_description
+
+
 class TimezoneOption(Option):
 
     class_description = \
@@ -639,6 +793,17 @@
         return value
 
 
+class HttpVersionOption(Option):
+    """Used by roundup-server to verify http version is set to valid
+       string."""
+
+    def str2value(self, value):
+        if value not in ["HTTP/1.0", "HTTP/1.1"]:
+            raise OptionValueError(self, value,
+                "Valid vaues for -V or --http_version are: HTTP/1.0, HTTP/1.1")
+        return value
+
+
 class RegExpOption(Option):
 
     """Regular Expression option (value is Regular Expression Object)"""
@@ -758,17 +923,20 @@
             "Force Roundup to use a particular text indexer.\n"
             "If no indexer is supplied, the first available indexer\n"
             "will be used in the following order:\n"
-            "Possible values: xapian, whoosh, native (internal)."),
+            "Possible values: xapian, whoosh, native (internal), "
+            "native-fts.\nNote 'native-fts' will only be used if set."),
         (Option, "indexer_language", "english",
             "Used to determine what language should be used by the\n"
-            "indexer above. Currently only affects Xapian indexer. It\n"
-            "sets the language for the stemmer.\n"
+            "indexer above. Applies to Xapian and PostgreSQL native-fts\n"
+            "indexer. It sets the language for the stemmer, and PostgreSQL\n"
+            "native-fts stopwords and other dictionaries.\n"
             "Possible values: must be a valid language for the indexer,\n"
             "see indexer documentation for details."),
         (WordListOption, "indexer_stopwords", "",
             "Additional stop-words for the full-text indexer specific to\n"
             "your tracker. See the indexer source for the default list of\n"
-            "stop-words (eg. A,AND,ARE,AS,AT,BE,BUT,BY, ...)"),
+            "stop-words (eg. A,AND,ARE,AS,AT,BE,BUT,BY, ...). This is\n"
+            "not used by the native-fts indexer."),
         (OctalNumberOption, "umask", "0o002",
             "Defines the file creation mode mask."),
         (IntegerNumberGeqZeroOption, 'csv_field_size', '131072',
@@ -1038,7 +1206,7 @@
             "Setting this option makes Roundup migrate passwords with\n"
             "an insecure password-scheme to a more secure scheme\n"
             "when the user logs in via the web-interface."),
-        (MandatoryOption, "secret_key", create_token(),
+        (SecretMandatoryOption, "secret_key", create_token(),
             "A per tracker secret used in etag calculations for\n"
             "an object. It must not be empty.\n"
             "It prevents reverse engineering hidden data in an object\n"
@@ -1050,7 +1218,7 @@
             "(Note the default value changes every time\n"
             "     roundup-admin updateconfig\n"
             "is run, so it must be explicitly set to a non-empty string.\n"),
-        (MandatoryOption, "jwt_secret", "disabled",
+        (SecretNullableOption, "jwt_secret", "disabled",
             "This is used to generate/validate json web tokens (jwt).\n"
             "Even if you don't use jwts it must not be empty.\n"
             "If less than 256 bits (32 characters) in length it will\n"
@@ -1076,7 +1244,7 @@
         (NullableOption, 'user', 'roundup',
             "Database user name that Roundup should use.",
             ['MYSQL_DBUSER']),
-        (NullableOption, 'password', 'roundup',
+        (SecretNullableOption, 'password', 'roundup',
             "Database user password.",
             ['MYSQL_DBPASSWORD']),
         (NullableOption, 'read_default_file', '~/.my.cnf',
@@ -1157,7 +1325,7 @@
         (Option, "username", "", "SMTP login name.\n"
             "Set this if your mail host requires authenticated access.\n"
             "If username is not empty, password (below) MUST be set!"),
-        (Option, "password", NODEFAULT, "SMTP login password.\n"
+        (SecretMandatoryOption, "password", NODEFAULT, "SMTP login password.\n"
             "Set this if your mail host requires authenticated access."),
         (IntegerNumberGeqZeroOption, "port", smtplib.SMTP_PORT,
             "Default port to send SMTP on.\n"
@@ -1400,7 +1568,11 @@
     # actual name of the config file.  set on load.
     filepath = os.path.join(HOME, INI_FILE)
 
-    def __init__(self, config_path=None, layout=None, settings={}):
+    # List of option names that need additional validation after
+    # all options are loaded.
+    option_validators = []
+
+    def __init__(self, config_path=None, layout=None, settings=None):
         """Initialize confing instance
 
         Parameters:
@@ -1417,6 +1589,8 @@
                 The overrides are applied after loading config file.
 
         """
+        if settings is None:
+            settings = {}
         # initialize option containers:
         self.sections = []
         self.section_descriptions = {}
@@ -1471,6 +1645,9 @@
         for _name in option.aliases:
             self.options[_name] = option
 
+        if hasattr(option, 'validate'):
+            self.option_validators.append(option.name)
+
     def update_option(self, name, klass,
                       default=NODEFAULT, description=None):
         """Override behaviour of early created option.
@@ -1815,7 +1992,9 @@
     ext = None
     detectors = None
 
-    def __init__(self, home_dir=None, settings={}):
+    def __init__(self, home_dir=None, settings=None):
+        if settings is None:
+            settings = {}
         Config.__init__(self, home_dir, layout=SETTINGS, settings=settings)
         # load the config if home_dir given
         if home_dir is None:
@@ -1882,25 +2061,10 @@
             on each other. E.G. indexer_language can only be
             validated if xapian indexer is used.
         """
-        if options['INDEXER']._value in ("", "xapian"):
-            try:
-                import xapian
-            except ImportError:
-                # indexer is probably '' and xapian isn't present
-                # so just return at end of method
-                pass
-            else:
-                try:
-                    lang = options["INDEXER_LANGUAGE"]._value
-                    xapian.Stem(lang)
-                except xapian.InvalidArgumentError:
-                    import textwrap
-                    lang_avail = b2s(xapian.Stem.get_available_languages())
-                    languages = textwrap.fill(_("Valid languages: ") +
-                                              lang_avail, 75,
-                                              subsequent_indent="   ")
-                    raise OptionValueError( options["INDEXER_LANGUAGE"],
-                                            lang, languages)
+
+        for option in self.option_validators:
+            # validate() should throw an exception if there is an issue.
+            options[option].validate(options)
 
     def load(self, home_dir):
         """Load configuration from path designated by home_dir argument"""