Return all iterations from getValues for values stored at every iteration

pyoptsparse/pyOpt_history.py

@@ -214,11 +214,19 @@ def _processDB(self) -> None:
         # extract all information stored in the call counters
         self.iterKeys = set()
         self.extraFuncsNames = set()
+        funcEvalKeys = None
         for i in self.callCounters:
             val = self.read(i)
             self.iterKeys.update(val.keys())
             if "funcs" in val.keys():
                 self.extraFuncsNames.update(val["funcs"].keys())
+                if funcEvalKeys is None:
+                    funcEvalKeys = set(val.keys())
+                else:
+                    funcEvalKeys.intersection_update(val.keys())
+        # keys that are missing from some function evaluations, such as the values
+        # SNOPT stores only at major iterations
+        self.majorIterKeys = self.iterKeys.difference(funcEvalKeys or set())
         # remove objective and constraint keys
         self.extraFuncsNames = self.extraFuncsNames.difference(self.conNames).difference(self.objNames)
 
@@ -231,14 +239,15 @@ def _processDB(self) -> None:
 
     def getIterKeys(self):
         """
-        Returns the keys available at each optimization iteration.
+        Returns the keys stored at the optimization iterations.
         This function is useful for inspecting the history file, to determine
         what information is saved at each iteration.
+        Some keys are only stored at major iterations, such as ``merit`` for SNOPT.
 
         Returns
         -------
         list of str
-            A list containing the names of keys stored at each optimization iteration.
+            A list containing the names of keys stored at any optimization iteration.
         """
         return copy.deepcopy(list(self.iterKeys))
 
@@ -516,6 +525,10 @@ def getValues(self, names=None, callCounters=None, major=True, scale=False, stac
 
         major : bool
             flag to specify whether to include only major iterations.
+            Some values are only stored at major iterations, such as ``merit`` for SNOPT.
+            If such a value is requested in `names` with ``major=False``, `major` is set to True.
+            If `names` is None, such values are left out instead, and the other values are
+            returned for all iterations.
 
         scale : bool
             flag to specify whether to apply scaling for the values. True means
@@ -572,6 +585,7 @@ def getValues(self, names=None, callCounters=None, major=True, scale=False, stac
             .union(self.extraFuncsNames)
             .difference({"funcs", "funcsSens", "xuser"})
         )
+        namesGiven = names is not None
         # cast string input into a single list
         if isinstance(names, str):
             names = {names}
@@ -597,12 +611,20 @@ def getValues(self, names=None, callCounters=None, major=True, scale=False, stac
                     + "Otherwise, consider renaming the variable or manually editing the history file."
                 )
 
-        if len(names.intersection(self.iterKeys)) > 0:
-            if not major:
+        majorOnlyNames = names.intersection(self.majorIterKeys)
+        if len(majorOnlyNames) > 0 and not major:
+            if namesGiven:
                 pyOptSparseWarning(
-                    "The major flag has been set to True, since some names specified only exist on major iterations."
+                    f"The major flag has been set to True, since the names {sorted(majorOnlyNames)} "
+                    + "only exist on major iterations."
                 )
                 major = True
+            else:
+                names = names.difference(majorOnlyNames)
+                pyOptSparseWarning(
+                    f"The names {sorted(majorOnlyNames)} only exist on major iterations "
+                    + "and are not returned, since the major flag is False."
+                )
 
         if stack:
             DVinNames = names.intersection(self.DVNames)

pyoptsparse/testing/pyOpt_testing.py

@@ -345,6 +345,8 @@ def check_hist_file(self, tol):
         times = hist.getValues(names="time", major=False)["time"]
         # the times should be monotonically increasing
         self.assertTrue(np.all(np.diff(times) > 0))
+        # all iterations are returned, not only the major ones
+        self.assertGreaterEqual(len(times), len(hist.getValues(names="time", major=True)["time"]))
 
         # this check is only used for optimizers that guarantee '0' and 'last' contain funcs
         if self.optName in ["SNOPT", "PSQP"]:

tests/test_hs015.py

@@ -7,6 +7,7 @@
 # External modules
 from baseclasses.utils import readPickle, writePickle
 import numpy as np
+from numpy.testing import assert_allclose
 from parameterized import parameterized
 
 # First party modules
@@ -222,6 +223,31 @@ def test_ipopt(self):
         for i in range(objhis_len - 1):
             self.assertNotEqual(data["obj"][i], data["obj"][i + 1])
 
+    @parameterized.expand([("SNOPT", "merit"), ("IPOPT", "mu"), ("Uno", "stationarity_residual")])
+    def test_history_minor_iterations(self, optName, majorOnlyKey):
+        self.optName = optName
+        self.setup_optProb()
+        self.optimize(storeHistory=True)
+        hist = History(self.histFileName, flag="r")
+        objMajor = hist.getValues(names="obj", major=True)["obj"]
+        objAll = hist.getValues(names="obj", major=False)["obj"]
+        self.assertGreater(len(objAll), len(objMajor))
+
+        # values stored at every iteration are returned for all iterations
+        data = hist.getValues(names=["time", "isMajor"], major=False)
+        self.assertEqual(len(data["time"]), len(objAll))
+        self.assertFalse(data["isMajor"].all())
+
+        # with names=None, values stored only at major iterations are left out
+        data = hist.getValues(major=False)
+        self.assertNotIn(majorOnlyKey, data)
+        assert_allclose(data["obj"], objAll)
+
+        # requesting a value stored only at major iterations sets major=True
+        data = hist.getValues(names=["obj", majorOnlyKey], major=False)
+        assert_allclose(data["obj"], objMajor)
+        self.assertEqual(len(data[majorOnlyKey]), len(objMajor))
+
     def test_snopt_hotstart(self):
         self.optName = "SNOPT"
         self.setup_optProb()