Merge branch 'trunk' into dict-for-data

README.md

@@ -32,7 +32,7 @@ The documentation can be found at [github.io](https://pdb-redo.github.io/libcifp
 
 ## Synopsis
 
-```c++
+```cpp
 // A simple program counting residues with an OXT atom
 
 #include <filesystem>

changelog

@@ -7,6 +7,7 @@ Version 7.0.10
 - Replace code creating quaternions from rotation matrices
   that might sometimes give incorrect results. Or at least,
   the test code failed on this particular kind of code. Sometimes.
+- Fix reconstruction to build pdbx_nonpoly_scheme
 
 Version 7.0.9
 - Using cif::file::load_dictionary it is now possible to

docs/index.rst

@@ -27,7 +27,7 @@ Using *libcifpp* is easy, if you are familiar with modern C++:
 
 .. literalinclude:: ../README.md
 	:language: c++
-	:start-after: ```c++
+	:start-after: ```cpp
 	:end-before: ```
 
 .. toctree::

include/cif++/category.hpp

@@ -178,7 +178,7 @@ class category
 	/// @brief Set the validator for this category to @a v
 	/// @param v The category_validator to assign. A nullptr value is allowed.
 	/// @param db The enclosing @ref datablock
-	void set_validator(const validator_base *v, datablock &db);
+	void set_validator(const validator *v, datablock &db);
 
 	/// @brief Update the links in this category
 	/// @param db The enclosing @ref datablock
@@ -186,7 +186,7 @@ class category
 
 	/// @brief Return the global @ref validator for the data
 	/// @return The @ref validator or nullptr if not assigned
-	const validator_base *get_validator() const { return m_validator; }
+	const validator *get_validator() const { return m_validator; }
 
 	/// @brief Return the category validator for this category
 	/// @return The @ref category_validator or nullptr if not assigned
@@ -1285,7 +1285,7 @@ class category
 
 	std::string m_name;
 	std::vector<item_entry> m_items;
-	const validator_base *m_validator = nullptr;
+	const validator *m_validator = nullptr;
 	const category_validator *m_cat_validator = nullptr;
 	std::vector<link> m_parent_links, m_child_links;
 	bool m_cascade = true;

include/cif++/datablock.hpp

@@ -116,14 +116,14 @@ class datablock : public std::list<category>
 	 * 
 	 * @param v The new validator object, may be null
 	 */
-	void set_validator(const validator_base *v);
+	void set_validator(const validator *v);
 
 	/**
 	 * @brief Get the validator object
 	 * 
 	 * @return const validator* The validator or nullptr if there is none
 	 */
-	const validator_base *get_validator() const;
+	const validator *get_validator() const;
 
 	/**
 	 * @brief Validates the content of this datablock and all its content
@@ -244,7 +244,7 @@ class datablock : public std::list<category>
 
   private:
 	std::string m_name;
-	const validator_base *m_validator = nullptr;
+	const validator *m_validator = nullptr;
 };
 
 } // namespace cif

include/cif++/dictionary_parser.hpp

@@ -42,4 +42,9 @@ namespace cif
  */
 validator parse_dictionary(std::string_view name, std::istream &is);
 
+/**
+ * @brief Extend the definitions in validator @a v with the contents of stream @a is
+ */
+void extend_dictionary(validator &v, std::istream &is);
+
 } // namespace cif

include/cif++/file.hpp

@@ -205,10 +205,10 @@ class file : public std::list<datablock>
 	void load(std::istream &is);
 
 	/** Load the data from the file specified by @a p using validator @a v */
-	void load(const std::filesystem::path &p, const validator_base &v);
+	void load(const std::filesystem::path &p, const validator &v);
 
 	/** Load the data from @a is using validator @a v */
-	void load(std::istream &is, const validator_base &v);
+	void load(std::istream &is, const validator &v);
 
 	/** Load the data from the file specified by @a p using a validator constructed from dictionary @a dict */
 	void load(const std::filesystem::path &p, std::string_view dict);

include/cif++/parser.hpp

@@ -39,7 +39,7 @@
 namespace cif
 {
 
-class validator_base;
+class validator;
 
 // --------------------------------------------------------------------
 
@@ -310,7 +310,7 @@ class parser : public sac_parser
 {
   public:
 	/// \brief constructor, generates data into @a file from @a is using validator @a v
-	parser(std::istream &is, file &file, const validator_base *v)
+	parser(std::istream &is, file &file, const validator *v)
 		: sac_parser(is)
 		, m_file(file)
 		, m_validator(v)
@@ -337,7 +337,7 @@ class parser : public sac_parser
 	file &m_file;
 	datablock *m_datablock = nullptr;
 	category *m_category = nullptr;
-	const validator_base *m_validator = nullptr;
+	const validator *m_validator = nullptr;
 	row_handle m_row;
 
 	/** @endcond */

include/cif++/validate.hpp

@@ -32,7 +32,6 @@
 #include <filesystem>
 #include <list>
 #include <mutex>
-#include <optional>
 #include <system_error>
 #include <utility>
 
@@ -49,7 +48,6 @@
 namespace cif
 {
 
-class category;
 struct category_validator;
 
 // --------------------------------------------------------------------
@@ -386,80 +384,11 @@ struct link_validator
 
 // --------------------------------------------------------------------
 
-class validator_base
-{
-  public:
-	/// @brief destructor
-	virtual ~validator_base() = default;
-
-	validator_base(const validator_base &rhs) = delete;
-	validator_base &operator=(const validator_base &rhs) = delete;
-
-	/// @brief move constructor
-	validator_base(validator_base &&rhs) = default;
-
-	/// @brief move assignment operator
-	validator_base &operator=(validator_base &&rhs) = default;
-
-	/// @brief Return the type validator for @a type_code, may return nullptr
-	virtual const type_validator *get_validator_for_type(std::string_view type_code) const = 0;
-
-	/// @brief Return the category validator for @a category, may return nullptr
-	virtual const category_validator *get_validator_for_category(std::string_view category) const = 0;
-
-	/// @brief Return the list of link validators for which the parent is @a category
-	virtual std::vector<const link_validator *> get_links_for_parent(std::string_view category) const = 0;
-
-	/// @brief Return the list of link validators for which the child is @a category
-	virtual std::vector<const link_validator *> get_links_for_child(std::string_view category) const = 0;
-
-	/// @brief Bottleneck function to report an error in validation
-	void report_error(validation_error err, bool fatal = true) const
-	{
-		report_error(make_error_code(err), fatal);
-	}
-
-	/// @brief Bottleneck function to report an error in validation
-	void report_error(std::error_code ec, bool fatal = true) const;
-
-	/// @brief Bottleneck function to report an error in validation
-	void report_error(validation_error err, std::string_view category,
-		std::string_view item, bool fatal = true) const
-	{
-		report_error(make_error_code(err), category, item, fatal);
-	}
-
-	/// @brief Bottleneck function to report an error in validation
-	void report_error(std::error_code ec, std::string_view category,
-		std::string_view item, bool fatal = true) const;
-
-	const std::string &name() const { return m_name; }       ///< Get the name of this validator
-	const std::string &version() const { return m_version; } ///< Get the version of this validator
-	bool is_strict() const { return m_strict; }              ///< Get the strict flag of this validator
-
-  protected:
-	/**
-	 * @brief Construct a new validator object
-	 *
-	 * @param name The name of the underlying dictionary
-	 */
-	validator_base(std::string_view name)
-		: m_name(name)
-	{
-	}
-
-	validator_base() = default;
-
-	std::string m_name;
-	std::string m_version;
-	bool m_strict = false;
-};
-
 /**
  * @brief The validator class combines all the link, category and item validator classes
  *
  */
-class validator : public validator_base
+class validator
 {
   public:
 	/**
@@ -468,7 +397,7 @@ class validator : public validator_base
 	 * @param name The name of the underlying dictionary
 	 */
 	validator(std::string_view name)
-		: validator_base(name)
+		: m_name(name)
 	{
 	}
 
@@ -490,30 +419,56 @@ class validator : public validator_base
 	void add_type_validator(type_validator &&v);
 
 	/// @brief Return the type validator for @a type_code, may return nullptr
-	const type_validator *get_validator_for_type(std::string_view type_code) const override;
+	const type_validator *get_validator_for_type(std::string_view type_code) const;
 
 	/// @brief Add category_validator @a v to the list of category validators
 	void add_category_validator(category_validator &&v);
 
 	/// @brief Return the category validator for @a category, may return nullptr
-	const category_validator *get_validator_for_category(std::string_view category) const override;
+	const category_validator *get_validator_for_category(std::string_view category) const;
 
 	/// @brief Add link_validator @a v to the list of link validators
 	void add_link_validator(link_validator &&v);
 
 	/// @brief Return the list of link validators for which the parent is @a category
-	std::vector<const link_validator *> get_links_for_parent(std::string_view category) const override;
+	std::vector<const link_validator *> get_links_for_parent(std::string_view category) const;
 
 	/// @brief Return the list of link validators for which the child is @a category
-	std::vector<const link_validator *> get_links_for_child(std::string_view category) const override;
+	std::vector<const link_validator *> get_links_for_child(std::string_view category) const;
 
-	void set_name(const std::string &name) { m_name = name; }             ///< Set the name of this validator
+	/// @brief Bottleneck function to report an error in validation
+	void report_error(validation_error err, bool fatal = true) const
+	{
+		report_error(make_error_code(err), fatal);
+	}
+
+	/// @brief Bottleneck function to report an error in validation
+	void report_error(std::error_code ec, bool fatal = true) const;
+
+	/// @brief Bottleneck function to report an error in validation
+	void report_error(validation_error err, std::string_view category,
+		std::string_view item, bool fatal = true) const
+	{
+		report_error(make_error_code(err), category, item, fatal);
+	}
+
+	/// @brief Bottleneck function to report an error in validation
+	void report_error(std::error_code ec, std::string_view category,
+		std::string_view item, bool fatal = true) const;
+
+	const std::string &name() const { return m_name; }        ///< Get the name of this validator
+	void set_name(const std::string &name) { m_name = name; } ///< Set the name of this validator
+
+	const std::string &version() const { return m_version; }              ///< Get the version of this validator
 	void set_version(const std::string &version) { m_version = version; } ///< Set the version of this validator
 
   private:
 	// name is fully qualified here:
 	item_validator *get_validator_for_item(std::string_view name) const;
 
+	std::string m_name;
+	std::string m_version;
+	bool m_strict = false;
 	std::set<type_validator> m_type_validators;
 	std::set<category_validator> m_category_validators;
 	std::vector<link_validator> m_link_validators;
@@ -521,46 +476,6 @@ class validator : public validator_base
 
 // --------------------------------------------------------------------
 
-class extended_validator : public validator_base
-{
-  public:
-	/**
-	 * @brief Construct a new validator object
-	 *
-	 * @param name The name of the underlying dictionary
-	 * @param validators The validators this extended validator is composed off
-	 */
-	extended_validator(std::vector<const validator *> validators);
-
-	extended_validator(const extended_validator &rhs) = delete;
-	extended_validator &operator=(const extended_validator &rhs) = delete;
-
-	/// @brief move constructor
-	extended_validator(extended_validator &&rhs) = default;
-
-	/// @brief move assignment operator
-	extended_validator &operator=(extended_validator &&rhs) = default;
-
-	/// @brief Return the type validator for @a type_code, may return nullptr
-	virtual const type_validator *get_validator_for_type(std::string_view type_code) const override;
-
-	/// @brief Return the category validator for @a category, may return nullptr
-	virtual const category_validator *get_validator_for_category(std::string_view category) const override;
-
-	/// @brief Return the list of link validators for which the parent is @a category
-	virtual std::vector<const link_validator *> get_links_for_parent(std::string_view category) const override;
-
-	/// @brief Return the list of link validators for which the child is @a category
-	virtual std::vector<const link_validator *> get_links_for_child(std::string_view category) const override;
-
-  protected:
-	friend class validator_factory;
-
-	std::vector<const validator *> m_validators;
-};
-
-// --------------------------------------------------------------------
-
 /**
  * @brief Validators are globally unique objects, use the validator_factory
  * class to construct them. This class is a singleton.
@@ -573,36 +488,18 @@ class validator_factory
 	static validator_factory &instance();
 
 	/// @brief Return the validator with name @a dictionary_name
-	[[deprecated("use construct_validator(const category &audit_conform) instead")]]
-	const validator_base &operator[](std::string_view dictionary_name);
-
-	/// @brief Return a validator for the data contained in an audit_conform category
-	const validator_base &construct_validator(const category &audit_conform);
-
-	/// @brief Construct a new validator with name @a name from resource data with at least version @a version if specified
-	const validator &construct_validator(std::string_view name,
-		std::optional<std::string> version);
-
-	/// @brief Construct a new validator with name @a name from the data in @a is with at least version @a version if specified
-	const validator &construct_validator(std::string_view name,
-		std::optional<std::string> version, std::istream &is);
+	const validator &operator[](std::string_view dictionary_name);
 
 	/// @brief Construct a new validator with name @a name from the data in @a is
-	const validator &construct_validator(std::string_view name, std::istream &is)
-	{
-		return construct_validator(name, {}, is);
-	}
+	const validator &construct_validator(std::string_view name, std::istream &is);
 
   private:
 	// --------------------------------------------------------------------
 
 	validator_factory() = default;
 
-	static bool check_version(std::string_view name, std::string_view expected, std::string_view found);
-
 	std::mutex m_mutex;
 	std::list<validator> m_validators;
-	std::list<extended_validator> m_extended_validators;
 };
 
 } // namespace cif

src/category.cpp

@@ -617,7 +617,7 @@ std::set<uint16_t> category::key_item_indices() const
 
 // --------------------------------------------------------------------
 
-void category::set_validator(const validator_base *v, datablock &db)
+void category::set_validator(const validator *v, datablock &db)
 {
 	m_validator = v;
 
@@ -696,7 +696,7 @@ bool category::is_valid() const
 	bool result = true;
 
 	if (m_validator == nullptr)
-		throw std::runtime_error("no validator specified");
+		throw std::runtime_error("no Validator specified");
 
 	if (empty())
 	{

src/compound.cpp

@@ -299,14 +299,37 @@ class compound_factory_impl : public std::enable_shared_from_this<compound_facto
 		std::shared_lock lock(mMutex);
 
 		compound *result = nullptr;
-		
+
+		// walk the list, see if any of the implementations has the compound already
 		for (auto impl = shared_from_this(); impl; impl = impl->m_next)
 		{
-			result = impl->create(id);
-			if (result != nullptr)
+			for (auto cmp : impl->m_compounds)
+			{
+				if (iequals(cmp->id(), id))
+				{
+					result = cmp;
+					break;
+				}
+			}
+
+			if (result)
 				break;
 		}
 
+		if (result == nullptr and
+			find_if(m_missing.begin(), m_missing.end(), [&id](const std::string &m_id) { return cif::iequals(id, m_id); }) == m_missing.end())
+		{
+			for (auto impl = shared_from_this(); impl; impl = impl->m_next)
+			{
+				result = impl->create(id);
+				if (result != nullptr)
+					break;
+			}
+
+			if (result == nullptr)
+				m_missing.emplace_back(id);
+		}
+
 		return result;
 	}
 
@@ -337,7 +360,7 @@ class compound_factory_impl : public std::enable_shared_from_this<compound_facto
 	cif::parser::datablock_index m_index;
 
 	std::vector<compound *> m_compounds;
-	cif::iset m_missing;
+	std::vector<std::string> m_missing;
 	std::shared_ptr<compound_factory_impl> m_next;
 };
 
@@ -358,14 +381,6 @@ compound_factory_impl::compound_factory_impl(const fs::path &file, std::shared_p
 
 compound *compound_factory_impl::create(const std::string &id)
 {
-	// shortcut
-
-	if (m_missing.contains(id))
-		return nullptr;
-
-	if (auto i = find_if(m_compounds.begin(), m_compounds.end(), [id](compound *c) { return c->id() == id; }); i != m_compounds.end())
-		return *i;
-
 	compound *result = nullptr;
 
 	std::unique_ptr<std::istream> ccd;
@@ -434,9 +449,6 @@ compound *compound_factory_impl::create(const std::string &id)
 		}
 	}
 
-	if (result == nullptr)
-		m_missing.insert(id);
-
 	return result;
 }
 
@@ -449,6 +461,20 @@ class local_compound_factory_impl : public compound_factory_impl
 		: compound_factory_impl(next)
 		, m_local_file(file)
 	{
+		// const std::regex peptideRx("(?:[lmp]-)?peptide", std::regex::icase);
+
+		// for (const auto &[id, name, threeLetterCode, group] :
+		// 	file["comp_list"]["chem_comp"].rows<std::string, std::string, std::string, std::string>("id", "name", "three_letter_code", "group"))
+		// {
+		// 	auto &rdb = m_local_file["comp_" + id];
+		// 	if (rdb.empty())
+		// 	{
+		// 		// std::cerr << "Missing data in restraint file for id " + id + '\n';
+		// 		continue;
+		// 	}
+
+		// 	construct_compound(rdb, id, name, threeLetterCode, group);
+		// }
 	}
 
 	compound *create(const std::string &id) override;
@@ -462,12 +488,6 @@ class local_compound_factory_impl : public compound_factory_impl
 
 compound *local_compound_factory_impl::create(const std::string &id)
 {
-	if (m_missing.contains(id))
-		return nullptr;
-
-	if (auto i = find_if(m_compounds.begin(), m_compounds.end(), [id](compound *c) { return c->id() == id; }); i != m_compounds.end())
-		return *i;
-
 	compound *result = nullptr;
 
 	for (auto &db : m_local_file)
@@ -494,9 +514,6 @@ compound *local_compound_factory_impl::create(const std::string &id)
 		}
 	}
 
-	if (result == nullptr)
-		m_missing.insert(id);
-
 	return result;
 }
 
@@ -509,13 +526,9 @@ compound *local_compound_factory_impl::construct_compound(const datablock &rdb,
 	int formal_charge = 0;
 	std::map<std::string,std::size_t> formula_data;
 
-	for (std::size_t ord = 1; const auto &[atom_id, type_symbol, type, charge, x, y, z, xi, yi, zi] :
-		rdb["chem_comp_atom"].rows<std::string, std::string, std::string, int,
-			std::optional<float>, std::optional<float>, std::optional<float>,
-			std::optional<float>, std::optional<float>, std::optional<float>>(
-			"atom_id", "type_symbol", "type", "charge",
-			"model_Cartn_x", "model_Cartn_y", "model_Cartn_z",
-			"pdbx_model_Cartn_x_ideal", "pdbx_model_Cartn_y_ideal", "pdbx_model_Cartn_z_ideal"))
+	for (std::size_t ord = 1; const auto &[atom_id, type_symbol, type, charge, x, y, z] :
+		rdb["chem_comp_atom"].rows<std::string, std::string, std::string, int, float, float, float>(
+			"atom_id", "type_symbol", "type", "charge", "x", "y", "z"))
 	{
 		auto atom = cif::atom_type_traits(type_symbol);
 		formula_weight += atom.weight();
@@ -527,9 +540,9 @@ compound *local_compound_factory_impl::construct_compound(const datablock &rdb,
 			{ "atom_id", atom_id },
 			{ "type_symbol", type_symbol },
 			{ "charge", charge },
-			{ "model_Cartn_x",  x.has_value() ? x : xi, 3 },
-			{ "model_Cartn_y",  y.has_value() ? y : yi, 3 },
-			{ "model_Cartn_z",  z.has_value() ? z : zi, 3 },
+			{ "model_Cartn_x",  x, 3 },
+			{ "model_Cartn_y",  y, 3 },
+			{ "model_Cartn_z",  z, 3 },
 			{ "pdbx_ordinal", ord++ }
 		});
 

src/datablock.cpp

@@ -41,7 +41,25 @@ datablock::datablock(const datablock &db)
 void datablock::load_dictionary()
 {
 	if (auto *audit_conform = get("audit_conform"); audit_conform and not audit_conform->empty())
-		set_validator(&validator_factory::instance().construct_validator(*audit_conform));
+	{
+		std::string name = audit_conform->front().get<std::string>("dict_name");
+
+		if (name == "mmcif_pdbx_v50")
+			name = "mmcif_pdbx.dic"; // we had a bug here in libcifpp...
+
+		if (not name.empty())
+		{
+			try
+			{
+				load_dictionary(name);
+			}
+			catch (const std::exception &ex)
+			{
+				if (VERBOSE)
+					std::cerr << "Failed to load dictionary " << std::quoted(name) << ": " << ex.what() << '\n';
+			}
+		}
+	}
 }
 
 void datablock::load_dictionary(std::string_view name)
@@ -49,7 +67,7 @@ void datablock::load_dictionary(std::string_view name)
 	set_validator(&validator_factory::instance()[name]);
 }
 
-void datablock::set_validator(const validator_base *v)
+void datablock::set_validator(const validator *v)
 {
 	m_validator = v;
 
@@ -65,7 +83,7 @@ void datablock::set_validator(const validator_base *v)
 	}
 }
 
-const validator_base *datablock::get_validator() const
+const validator *datablock::get_validator() const
 {
 	return m_validator;
 }
@@ -90,7 +108,7 @@ bool datablock::is_valid()
 	bool result = true;
 	for (auto &cat : *this)
 		result = cat.is_valid() and result;
-
+	
 	// Add or remove the audit_conform block here.
 	if (result)
 	{
@@ -111,9 +129,7 @@ bool datablock::is_valid()
 		}
 	}
 	else
-		erase(std::find_if(begin(), end(), [](category &cat)
-				  { return cat.name() == "audit_conform"; }),
-			end());
+		erase(std::find_if(begin(), end(), [](category &cat) { return cat.name() == "audit_conform"; }), end());
 
 	return result;
 }
@@ -187,7 +203,7 @@ std::tuple<datablock::iterator, bool> datablock::emplace(std::string_view name)
 
 	if (is_new)
 	{
-		i = insert(end(), { name });
+		i = insert(end(), {name});
 		i->set_validator(m_validator, *this);
 	}
 
@@ -248,7 +264,7 @@ namespace
 		return std::get<2>(*i);
 	}
 
-	void calculate_cat_order(cat_order_t &cat_order, iter_t i, const validator_base &validator)
+	void calculate_cat_order(cat_order_t &cat_order, iter_t i, const validator &validator)
 	{
 		if (i == cat_order.end() or get_count(i) >= 0)
 			return;
@@ -321,7 +337,7 @@ void datablock::write(std::ostream &os) const
 	else
 	{
 		// mmcif support, sort of. First write the 'entry' Category
-		// and if it exists, _AND_ we have a validator, write out the
+		// and if it exists, _AND_ we have a Validator, write out the
 		// audit_conform record.
 
 		if (auto entry = get("entry"); entry != nullptr)

src/dictionary_parser.cpp

@@ -488,4 +488,11 @@ validator parse_dictionary(std::string_view name, std::istream &is)
 	return result;
 }
 
+void extend_dictionary(validator &v, std::istream &is)
+{
+	file f;
+	dictionary_parser p(v, is, f);
+	p.load_dictionary();
+}
+
 } // namespace cif

src/file.cpp

@@ -176,7 +176,7 @@ void file::load(std::istream &is, std::string_view dict)
 	load(is, validator_factory::instance().operator[](dict));
 }
 
-void file::load(const std::filesystem::path &p, const validator_base &v)
+void file::load(const std::filesystem::path &p, const validator &v)
 {
 	gzio::ifstream in(p);
 	if (not in.is_open())
@@ -192,7 +192,7 @@ void file::load(const std::filesystem::path &p, const validator_base &v)
 	}
 }
 
-void file::load(std::istream &is, const validator_base &v)
+void file::load(std::istream &is, const validator &v)
 {
 	parser p(is, *this);
 	p.parse_file();

src/model.cpp

@@ -1306,9 +1306,6 @@ structure::structure(datablock &db, std::size_t modelNr, StructureOpenOptions op
 	: m_db(db)
 	, m_model_nr(modelNr)
 {
-	if (db.get_validator() == nullptr)
-		db.load_dictionary();
-
 	auto &atomCat = db["atom_site"];
 
 	load_atoms_for_model(options);

src/pdb/reconstruct.cpp

@@ -1109,6 +1109,87 @@ void comparePolySeqSchemes(datablock &db)
 	}
 }
 
+void createPdbxEntityNonpoly(datablock &db)
+{
+	using namespace literals;
+
+	auto &atom_site = db["atom_site"];
+	auto &entity = db["entity"];
+	auto &pdbx_entity_nonpoly = db["pdbx_entity_nonpoly"];
+
+	for (const auto &[entity_id, type] : entity.find<std::string, std::string>("type"_key == "water" or "type"_key == "non-polymer", "id", "type"))
+	{
+		for (auto comp_id : atom_site.find<std::string>("label_entity_id"_key == entity_id, "label_comp_id"))
+		{
+			if (auto test_comp_id = pdbx_entity_nonpoly.find_first<std::optional<std::string>>("entity_id"_key == entity_id, "comp_id"); test_comp_id.has_value())
+			{
+				if (*test_comp_id != comp_id)
+					throw std::runtime_error("Inconsistent pdbx_entity_nonpoly record for entity " + entity_id + ", expected comp_id " + comp_id);
+
+				continue;
+			}
+
+			if (type == "water")
+				pdbx_entity_nonpoly.emplace({ //
+					{ "entity_id", entity_id },
+					{ "name", "water" },
+					{ "comp_id", comp_id }
+				});
+			else
+			{
+				auto c = cif::compound_factory::instance().create(comp_id);
+
+				std::string name = c ? c->name() : ".";
+
+				pdbx_entity_nonpoly.emplace({ //
+					{ "entity_id", entity_id },
+					{ "name", name },
+					{ "comp_id", comp_id }
+				});
+			}
+		}
+	}
+}
+
+void createPdbxNonpolyScheme(datablock &db)
+{
+	using namespace literals;
+
+	createPdbxEntityNonpoly(db);
+
+	auto &pdbx_entity_nonpoly = db["pdbx_entity_nonpoly"];
+	auto &pdbx_nonpoly_scheme = db["pdbx_nonpoly_scheme"];
+	auto &atom_site = db["atom_site"];
+
+	for (const auto &[entity_id, comp_id] : pdbx_entity_nonpoly.rows<std::string,std::string>("entity_id", "comp_id"))
+	{
+		for (int ndb_nr = 1; auto row : atom_site.find("label_entity_id"_key == entity_id and "label_comp_id"_key == comp_id))
+		{
+			// Skip existing records
+			auto linked = atom_site.get_linked(row, pdbx_nonpoly_scheme);
+			if (not linked.empty())
+				continue;
+
+			int num = row.get<int>("auth_seq_id");
+
+			pdbx_nonpoly_scheme.emplace({//
+
+				{ "asym_id", row.get<std::string>("label_asym_id") },
+				{ "entity_id", entity_id },
+				{ "mon_id", comp_id },
+				{ "ndb_seq_num", ndb_nr++ },
+				{ "pdb_seq_num", num },
+				{ "auth_seq_num", num },
+				{ "pdb_mon_id", row.get<std::string>("auth_comp_id") },
+				{ "auth_mon_id",  row.get<std::string>("auth_comp_id") },
+				{ "pdb_strand_id", row.get<std::string>("auth_asym_id") },
+				{ "pdb_ins_code", row.get<std::string>("pdbx_PDB_ins_code") }
+
+			});
+		}
+	}
+}
+
 bool reconstruct_pdbx(file &file, std::string_view dictionary)
 {
 	if (file.empty())
@@ -1396,6 +1477,8 @@ bool reconstruct_pdbx(file &file, std::string_view dictionary)
 
 	if (auto cat = db.get("ndb_poly_seq_scheme"); cat == nullptr or cat->empty())
 		comparePolySeqSchemes(db);
+	
+	createPdbxNonpolyScheme(db);
 
 	// skip unknown categories for now
 	bool valid = true;

src/pdb/validate-pdbx.cpp

@@ -29,7 +29,7 @@
 namespace cif::pdb
 {
 
-condition get_parents_condition(const validator_base &validator, row_handle rh, const category &parentCat)
+condition get_parents_condition(const validator &validator, row_handle rh, const category &parentCat)
 {
 	condition result;
 

src/validate.cpp

@@ -25,7 +25,6 @@
  */
 
 #include "cif++/validate.hpp"
-#include "cif++/category.hpp"
 #include "cif++/dictionary_parser.hpp"
 #include "cif++/gzio.hpp"
 #include "cif++/utilities.hpp"
@@ -285,27 +284,6 @@ const item_validator *category_validator::get_validator_for_aliased_item(std::st
 
 // --------------------------------------------------------------------
 
-void validator_base::report_error(std::error_code ec, bool fatal) const
-{
-	if (m_strict or fatal)
-		throw validation_exception(ec);
-	else if (VERBOSE > 0)
-		std::cerr << ec.message() << '\n';
-}
-
-void validator_base::report_error(std::error_code ec, std::string_view category,
-	std::string_view item, bool fatal) const
-{
-	auto ex = item.empty() ? validation_exception(ec, category) : validation_exception(ec, category, item);
-
-	if (m_strict or fatal)
-		throw ex;
-	else if (VERBOSE > 0)
-		std::cerr << ex.what() << '\n';
-}
-
-// --------------------------------------------------------------------
-
 void validator::add_type_validator(type_validator &&v)
 {
 	auto r = m_type_validators.insert(std::move(v));
@@ -419,76 +397,23 @@ std::vector<const link_validator *> validator::get_links_for_child(std::string_v
 	return result;
 }
 
-// --------------------------------------------------------------------
-
-extended_validator::extended_validator(std::vector<const validator *> validators)
-	: m_validators(validators)
+void validator::report_error(std::error_code ec, bool fatal) const
 {
-	std::vector<std::string> names, versions;
-
-	for (auto v : m_validators)
-	{
-		names.emplace_back(v->name());
-		versions.emplace_back(v->version());
-		m_strict = m_strict or v->is_strict();
-	}
-
-	m_name = cif::join(names, "; ");
-	m_version = cif::join(versions, "; ");
+	if (m_strict or fatal)
+		throw validation_exception(ec);
+	else if (VERBOSE > 0)
+		std::cerr << ec.message() << '\n';
 }
 
-const type_validator *extended_validator::get_validator_for_type(std::string_view type_code) const
+void validator::report_error(std::error_code ec, std::string_view category,
+	std::string_view item, bool fatal) const
 {
-	const type_validator *result = nullptr;
+	auto ex = item.empty() ? validation_exception(ec, category) : validation_exception(ec, category, item);
 
-	for (auto v : m_validators)
-	{
-		result = v->get_validator_for_type(type_code);
-		if (result)
-			break;
-	}
-
-	return result;
-}
-
-const category_validator *extended_validator::get_validator_for_category(std::string_view category) const
-{
-	const category_validator *result = nullptr;
-
-	for (auto v : m_validators)
-	{
-		result = v->get_validator_for_category(category);
-		if (result)
-			break;
-	}
-
-	return result;
-}
-
-std::vector<const link_validator *> extended_validator::get_links_for_parent(std::string_view category) const
-{
-	std::vector<const link_validator *> result;
-
-	for (auto v : m_validators)
-	{
-		auto links = v->get_links_for_parent(category);
-		result.insert(result.end(), links.begin(), links.end());
-	}
-
-	return result;
-}
-
-std::vector<const link_validator *> extended_validator::get_links_for_child(std::string_view category) const
-{
-	std::vector<const link_validator *> result;
-
-	for (auto v : m_validators)
-	{
-		auto links = v->get_links_for_child(category);
-		result.insert(result.end(), links.begin(), links.end());
-	}
-
-	return result;
+	if (m_strict or fatal)
+		throw ex;
+	else if (VERBOSE > 0)
+		std::cerr << ex.what() << '\n';
 }
 
 // --------------------------------------------------------------------
@@ -499,172 +424,113 @@ validator_factory &validator_factory::instance()
 	return s_instance;
 }
 
-const validator_base &validator_factory::operator[](std::string_view dictionary_name)
+const validator &validator_factory::operator[](std::string_view dictionary_name)
 {
-	category audit_conform("audit_conform");
-	for (auto part : cif::split(dictionary_name, ";", true))
-		audit_conform.emplace({ { "dict_name", part } });
-
-	return construct_validator(audit_conform);
-}
-
-const validator_base &validator_factory::construct_validator(const category &audit_conform)
-{
-	if (audit_conform.empty())
-		throw std::runtime_error("Empty audit_conform category, cannot create a validator");
-
-	std::lock_guard lock(m_mutex);
-
-	std::vector<const validator *> validators;
-
-	for (const auto &[name, version] : audit_conform.rows<std::string, std::optional<std::string>>("dict_name", "dict_version"))
+	try
 	{
-		auto &v = construct_validator(name, version);
-		validators.emplace_back(&v);
-	}
+		std::lock_guard lock(m_mutex);
 
-	if (validators.size() == 1)
-		return *validators.front();
-
-	// override mode, last dictionary is most important
-	std::reverse(validators.begin(), validators.end());
-		
-	for (auto &ev : m_extended_validators)
-	{
-		if (ev.m_validators == validators)
-			return ev;
-	}
-
-	return m_extended_validators.emplace_back(validators);
-}
-
-const validator &validator_factory::construct_validator(std::string_view name,
-	std::optional<std::string> version)
-{
-	for (auto &v : m_validators)
-	{
-		if (version.has_value())
-			check_version(name, *version, v.version());
-
-		if (v.name() == name)
-			return v;
-	}
-
-	std::filesystem::path dictionary(name);
-
-	auto data = load_resource(name);
-
-	if (not data and dictionary.extension().string() != ".dic")
-		data = load_resource(dictionary.parent_path() / (dictionary.filename().string() + ".dic"));
-
-	if (not data)
-	{
-		std::error_code ec;
-
-		// might be a compressed dictionary on disk
-		std::filesystem::path p = dictionary;
-		if (p.extension() == ".dic")
-			p = p.parent_path() / (p.filename().string() + ".gz");
-		else
-			p = p.parent_path() / (p.filename().string() + ".dic.gz");
-
-#if defined(CACHE_DIR) or defined(DATA_DIR)
-		if (not std::filesystem::exists(p, ec) or ec)
+		for (auto &validator : m_validators)
 		{
-			for (const char *dir : {
-# if defined(CACHE_DIR)
-					 CACHE_DIR,
-# endif
-# if defined(DATA_DIR)
-					 DATA_DIR
-# endif
-				 })
+			if (iequals(validator.name(), dictionary_name))
+				return validator;
+		}
+
+		// not found, try to see if it helps if we tweak the name a little
+
+		// too bad clang version 10 did not have a constructor for std::filesystem::path that accepts a std::string_view
+		std::filesystem::path dictionary(dictionary_name.data(), dictionary_name.data() + dictionary_name.length());
+
+		if (dictionary.extension() != ".dic")
+		{
+			auto dict_name = dictionary.filename().string() + ".dic";
+
+			for (auto &validator : m_validators)
 			{
-				auto p2 = std::filesystem::path(dir) / p;
-				if (std::filesystem::exists(p2, ec) and not ec)
-				{
-					swap(p, p2);
-					break;
-				}
+				if (iequals(validator.name(), dict_name))
+					return validator;
 			}
 		}
+
+		// not found, add it
+
+		validator v(dictionary_name);
+
+		for (bool first = true; auto part_name : cif::split(dictionary_name, ";", true))
+		{
+			auto data = load_resource(part_name);
+			dictionary.assign(part_name.begin(), part_name.end());
+
+			if (not data and dictionary.extension().string() != ".dic")
+				data = load_resource(dictionary.parent_path() / (dictionary.filename().string() + ".dic"));
+
+			if (not data)
+			{
+				std::error_code ec;
+
+				// might be a compressed dictionary on disk
+				std::filesystem::path p = dictionary;
+				if (p.extension() == ".dic")
+					p = p.parent_path() / (p.filename().string() + ".gz");
+				else
+					p = p.parent_path() / (p.filename().string() + ".dic.gz");
+
+#if defined(CACHE_DIR) or defined(DATA_DIR)
+				if (not std::filesystem::exists(p, ec) or ec)
+				{
+					for (const char *dir : {
+# if defined(CACHE_DIR)
+							 CACHE_DIR,
+# endif
+# if defined(DATA_DIR)
+							 DATA_DIR
+# endif
+						 })
+					{
+						auto p2 = std::filesystem::path(dir) / p;
+						if (std::filesystem::exists(p2, ec) and not ec)
+						{
+							swap(p, p2);
+							break;
+						}
+					}
+				}
 #endif
 
-		if (std::filesystem::exists(p, ec) and not ec)
-		{
-			auto in = std::make_unique<gzio::ifstream>(p);
+				if (std::filesystem::exists(p, ec) and not ec)
+				{
+					auto in = std::make_unique<gzio::ifstream>(p);
 
-			if (not in->is_open())
-				throw std::runtime_error("Could not open dictionary (" + p.string() + ")");
+					if (not in->is_open())
+						throw std::runtime_error("Could not open dictionary (" + p.string() + ")");
 
-			data.reset(in.release());
+					data.reset(in.release());
+				}
+				else
+					throw std::runtime_error("Dictionary not found or defined (" + dictionary.string() + ")");
+			}
+
+			if (std::exchange(first, false))
+				v = parse_dictionary(part_name, *data);
+			else
+				extend_dictionary(v, *data);
 		}
-		else
-			throw std::runtime_error("Dictionary not found or defined (" + dictionary.string() + ")");
-	}
 
-	return construct_validator(name, version, *data);
+		m_validators.emplace_back(std::move(v));
+
+		return m_validators.back();
+	}
+	catch (const std::exception &ex)
+	{
+		std::string msg = "Error while loading dictionary ";
+		msg += dictionary_name;
+		std::throw_with_nested(std::runtime_error(msg));
+	}
 }
 
-const validator &validator_factory::construct_validator(std::string_view name,
-	std::optional<std::string> version, std::istream &is)
+const validator &validator_factory::construct_validator(std::string_view name, std::istream &is)
 {
-	auto v = parse_dictionary(name, is);
-
-	if (version.has_value() and VERBOSE >= 0)
-	{
-		auto vv = v.version();
-
-		if (vv.empty())
-			std::clog << "Could not check version of dictionary " << name << " since this info is missing\n";
-		else
-			check_version(name, *version, vv);
-	}
-
-	return m_validators.emplace_back(std::move(v));
-}
-
-bool validator_factory::check_version(std::string_view name, std::string_view expected, std::string_view found)
-{
-	bool result = true;
-	auto el = cif::split(expected, ".");
-	auto fl = cif::split(found, ".");
-
-	auto eli = el.begin();
-	auto fli = fl.begin();
-
-	while (eli != el.end() and fli != fl.end())
-	{
-		int e_int, f_int;
-		if (auto [ptr, ec] = std::from_chars(eli->begin(), eli->end(), e_int); ec != std::errc{})
-		{
-			std::clog << "Could not parse requested version string for dictionary " << std::quoted(expected) << "\n";
-			result = false;
-			break;
-		}
-
-		if (auto [ptr, ec] = std::from_chars(fli->begin(), fli->end(), f_int); ec != std::errc{})
-		{
-			std::clog << "Could not parse version string in dictionary " << name << " " << std::quoted(found) << "\n";
-			result = false;
-			break;
-		}
-
-		if (f_int > e_int) // newer version, assume this is ok
-			break;
-
-		if (f_int < e_int)
-		{
-			std::clog << "The version in dictionary " << name << " is lower than requested, this may cause validation errors\n";
-			result = false;
-			break;
-		}
-
-		++eli;
-		++fli;
-	}
-
-	return result;
+	return m_validators.emplace_back(parse_dictionary(name, is));
 }
 
 } // namespace cif

test/unit-v2-test.cpp

@@ -3245,19 +3245,17 @@ _cat_1.name
 	std::istream is_data(&data_buffer);
 	f.load(is_data, validator);
 
-	CHECK(f.is_valid());
+	REQUIRE(f.is_valid());
 
 	std::stringstream ss;
 	ss << f;
 
 	cif::file f2(ss);
-	REQUIRE(f2.empty() == false);
-	f2.front().load_dictionary();
-	CHECK(f2.is_valid());
+	REQUIRE(f2.is_valid());
 
 	auto &audit_conform = f2.front()["audit_conform"];
-	CHECK(audit_conform.front()["dict_name"].as<std::string>() == "test_dict.dic");
-	CHECK(audit_conform.front()["dict_version"].as<float>() == 1.0);
+	REQUIRE(audit_conform.front()["dict_name"].as<std::string>() == "test_dict.dic");
+	REQUIRE(audit_conform.front()["dict_version"].as<float>() == 1.0);
 }
 
 // --------------------------------------------------------------------