nest.map_structure implemented and tested

henon · henon · commit ade4ef7f6530 · 2019-04-12T17:11:17.000+02:00
diff --git a/src/TensorFlowNET.Core/Python.cs b/src/TensorFlowNET.Core/Python.cs
@@ -120,6 +120,55 @@ public static float time()
                 yield return (t1.Data<T1>(i), t2.Data<T2>(i));
         }
 
+        public static IEnumerable<(T1, T2)> zip<T1, T2>(IEnumerable<T1> e1, IEnumerable<T2> e2)
+        {
+            var iter2 = e2.GetEnumerator();
+            foreach (var v1 in e1)
+            {
+                iter2.MoveNext();
+                var v2 = iter2.Current;
+                yield return (v1, v2);
+            }
+        }
+
+        /// <summary>
+        /// Untyped implementation of zip for arbitrary data
+        /// 
+        /// Converts an list of lists or arrays [[1,2,3], [4,5,6], [7,8,9]] into a list of arrays 
+        /// representing tuples of the same index of all source arrays [[1,4,7], [2,5,9], [3,6,9]]
+        /// </summary>
+        /// <param name="lists">one or multiple sequences to be zipped</param>
+        /// <returns></returns>
+        public static IEnumerable<object[]> zip(params object[] lists)
+        {
+            if (lists.Length == 0)
+                yield break;
+            var first = lists[0];
+            if (first == null)
+                yield break;
+            var arity = (first as IEnumerable).OfType<object>().Count();
+            for (int i = 0; i < arity; i++)
+            {
+                var array= new object[lists.Length];
+                for (int j = 0; j < lists.Length; j++)
+                    array[j] = GetSequenceElementAt(lists[j], i);
+                yield return array;
+            }
+        }
+
+        private static object GetSequenceElementAt(object sequence, int i)
+        {
+            switch (sequence)
+            {
+                case Array array:
+                    return array.GetValue(i);
+                case IList list:
+                    return list[i];
+                default:
+                    return (sequence as IEnumerable).OfType<object>().Skip(Math.Max(0, i)).FirstOrDefault();
+            }
+        }
+
         public static IEnumerable<(int, T)> enumerate<T>(IList<T> values)
         {
             for (int i = 0; i < values.Count; i++)
@@ -137,6 +186,7 @@ public static Dictionary<string, object> ConvertToDict(object dyn)
             }
             return dictionary;
         }
+
     }
 
     public interface IPython : IDisposable
diff --git a/src/TensorFlowNET.Core/Util/nest.py.cs b/src/TensorFlowNET.Core/Util/nest.py.cs
@@ -2,7 +2,6 @@
 using System.Collections;
 using System.Collections.Generic;
 using System.Linq;
-using System.Text;
 using NumSharp;
 
 namespace Tensorflow.Util
@@ -24,6 +23,14 @@ namespace Tensorflow.Util
     public static class nest
     {
 
+        public static IEnumerable<object[]> zip(params object[] structures)
+            => Python.zip(structures);
+
+        public static IEnumerable<(T1, T2)> zip<T1, T2>(IEnumerable<T1> e1, IEnumerable<T2> e2)
+            => Python.zip(e1, e2);
+
+        public static Dictionary<string, object> ConvertToDict(object dyn)
+            => Python.ConvertToDict(dyn);
 
         //def _get_attrs_values(obj):
         //  """Returns the list of values from an attrs instance."""
@@ -75,8 +82,14 @@ private static object _sequence_like(object instance, IEnumerable<object> args)
                 //# instances. This is intentional, to avoid potential bugs caused by mixing
                 //# ordered and plain dicts (e.g., flattening a dict but using a
                 //# corresponding `OrderedDict` to pack it back).
-                //    result = dict(zip(_sorted(instance), args))
-                //    return type(instance)((key, result[key]) for key in _six.iterkeys(instance))
+                switch (instance)
+                {
+                    case Hashtable hash:
+                        var result = new Hashtable();
+                        foreach ((object key, object value) in zip(_sorted(hash).OfType<object>(), args))
+                            result[key] = value;
+                        return result;
+                }
             }
             //else if( _is_namedtuple(instance) || _is_attrs(instance)) 
             //    return type(instance)(*args)
@@ -140,7 +153,9 @@ private static IEnumerable<object> _yield_value(object iterable)
         }
 
         //# See the swig file (util.i) for documentation.
-        public static bool is_sequence(object arg) => arg is IEnumerable && !(arg is string);
+        public static bool is_sequence(object arg) 
+            => arg is IEnumerable && !(arg is string) && !(arg is NDArray) &&
+                    !(arg.GetType().IsGenericType && arg.GetType().GetGenericTypeDefinition() == typeof(HashSet<>));
 
         public static bool is_mapping(object arg) => arg is IDictionary;
 
@@ -355,38 +370,54 @@ private static (int new_index, List<object> child) _packed_nest_with_indices(obj
         /// <returns> `flat_sequence` converted to have the same recursive structure as
         /// `structure`.
         /// </returns>
-        public static object pack_sequence_as(object structure, List<object> flat_sequence)
+        public static object pack_sequence_as<T>(object structure, IEnumerable<T> flat_sequence)
         {
-            if (flat_sequence == null)
+            List<object> flat = null;
+            if (flat_sequence is List<object>)
+                flat = flat_sequence as List<object>;
+            else 
+                flat=new List<object>(flat_sequence.OfType<object>());
+            if (flat_sequence==null)
                 throw new ArgumentException("flat_sequence must not be null");
             //  if not is_sequence(flat_sequence):
             //    raise TypeError("flat_sequence must be a sequence")
 
             if (!is_sequence(structure))
             {
-                if (len(flat_sequence) != 1)
-                    throw new ValueError($"Structure is a scalar but len(flat_sequence) ==  {len(flat_sequence)} > 1");
-                return flat_sequence.FirstOrDefault();
+                if (len(flat) != 1)
+                    throw new ValueError($"Structure is a scalar but len(flat_sequence) ==  {len(flat)} > 1");
+                return flat.FirstOrDefault();
             }
             int final_index = 0;
             List<object> packed = null;
             try
             {
-                (final_index, packed) = _packed_nest_with_indices(structure, flat_sequence, 0);
-                if (final_index < len(flat_sequence))
-                    throw new IndexOutOfRangeException($"Final index: { final_index} was smaller than  len(flat_sequence): { len(flat_sequence) }");
+                (final_index, packed) = _packed_nest_with_indices(structure, flat, 0);
+                if (final_index < len(flat))
+                    throw new IndexOutOfRangeException(
+                        $"Final index: {final_index} was smaller than  len(flat_sequence): {len(flat)}");
+                return _sequence_like(structure, packed);
             }
             catch (IndexOutOfRangeException)
             {
                 var flat_structure = flatten(structure);
-                if (len(flat_structure) != len(flat_sequence))
+                if (len(flat_structure) != len(flat))
                 {
                     throw new ValueError("Could not pack sequence. Structure had %d elements, but " +
-                                         $"flat_sequence had {len(flat_structure)} elements. flat_sequence had: {len(flat_sequence)}");
+                                         $"flat_sequence had {len(flat_structure)} elements. flat_sequence had: {len(flat)}");
+                }
+                return _sequence_like(structure, packed);
+            }
+            catch (ArgumentOutOfRangeException)
+            {
+                var flat_structure = flatten(structure);
+                if (len(flat_structure) != len(flat))
+                {
+                    throw new ValueError("Could not pack sequence. Structure had %d elements, but " +
+                                         $"flat_sequence had {len(flat_structure)} elements. flat_sequence had: {len(flat)}");
                 }
                 return _sequence_like(structure, packed);
             }
-            return packed;
         }
 
         /// <summary>
@@ -396,10 +427,9 @@ public static object pack_sequence_as(object structure, List<object> flat_sequen
         ///  `structure[i]`.  All structures in `structure` must have the same arity,
         ///  and the return value will contain the results in the same structure.
         /// </summary>
-        /// <typeparam name="T"></typeparam>
-        /// <typeparam name="U"></typeparam>
+        /// <typeparam name="T">the type of the elements of the output structure (object if diverse)</typeparam>
         /// <param name="func"> A callable that accepts as many arguments as there are structures.</param>
-        /// <param name="structure">scalar, or tuple or list of constructed scalars and/or other
+        /// <param name="structures">scalar, or tuple or list of constructed scalars and/or other
         ///      tuples/lists, or scalars.  Note: numpy arrays are considered as scalars.</param>
         /// <param name="check_types">If set to
         ///      `True` (default) the types of iterables within the structures have to be
@@ -414,18 +444,41 @@ public static object pack_sequence_as(object structure, List<object> flat_sequen
         ///    `check_types` is `False` the sequence types of the first structure will be
         ///    used.
         /// </returns>
-        public static IEnumerable<U> map_structure<T, U>(Func<T, U> func, IEnumerable<T> structure, bool check_types = false)
+        public static IEnumerable<object> map_structure(Func<object[], object> func, object structure, params object[] more_structures)
         {
+            // TODO: check structure and types
+            //  for other in structure[1:]:
+            //    assert_same_structure(structure[0], other, check_types=check_types)
+
+            if (more_structures.Length==0)
+            {
+                // we don't need to zip if we have only one structure
+                return map_structure(a => func(new object[]{a}), structure);
+            }
+            var flat_structures = new List<object>() { flatten(structure) };
+            flat_structures.AddRange(more_structures.Select(flatten));
+            var entries = zip(flat_structures);
+            var mapped_flat_structure = entries.Select(func);
 
+            return (pack_sequence_as(structure, mapped_flat_structure) as IEnumerable).OfType<object>();
+        }
+
+        /// <summary>
+        /// Same as map_structure, but with only one structure (no combining of multiple structures)
+        /// </summary>
+        /// <param name="func"></param>
+        /// <param name="structure"></param>
+        /// <returns></returns>
+        public static IEnumerable<object> map_structure(Func<object, object> func, object structure)
+        {
+            // TODO: check structure and types
             //  for other in structure[1:]:
             //    assert_same_structure(structure[0], other, check_types=check_types)
 
-            //  flat_structure = [flatten(s) for s in structure]
-            //  entries = zip(*flat_structure)
+            var flat_structure = flatten(structure);
+            var mapped_flat_structure = flat_structure.Select(func).ToList();
 
-            //  return pack_sequence_as(
-            //      structure[0], [func(*x) for x in entries])
-            return null;
+            return (pack_sequence_as(structure, mapped_flat_structure) as IEnumerable).OfType<object>();
         }
 
         //def map_structure_with_paths(func, *structure, **kwargs):
diff --git a/test/TensorFlowNET.UnitTest/PythonTest.cs b/test/TensorFlowNET.UnitTest/PythonTest.cs
@@ -4,6 +4,8 @@
 using System.Linq;
 using System.Text;
 using Microsoft.VisualStudio.TestTools.UnitTesting;
+using Newtonsoft.Json.Linq;
+using NumSharp;
 using Tensorflow;
 using Tensorflow.Util;
 
@@ -25,17 +27,45 @@ protected object None {
 
         public void assertItemsEqual(ICollection given, ICollection expected)
         {
+            if (given is Hashtable && expected is Hashtable)
+            {
+                Assert.AreEqual(JObject.FromObject(expected).ToString(), JObject.FromObject(given).ToString());
+                return;
+            }
             Assert.IsNotNull(expected);
             Assert.IsNotNull(given);
             var e = expected.OfType<object>().ToArray();
             var g = given.OfType<object>().ToArray();
             Assert.AreEqual(e.Length, g.Length, $"The collections differ in length expected {e.Length} but got {g.Length}");
             for (int i = 0; i < e.Length; i++)
-                Assert.AreEqual(e[i], g[i], $"Items differ at index {i}, expected {e[i]} but got {g[i]}");
+            {
+                if (g[i] is NDArray && e[i] is NDArray)
+                    assertItemsEqual((g[i] as NDArray).Array, (e[i] as NDArray).Array);
+                else if (e[i] is ICollection && g[i] is ICollection)
+                    assertEqual(g[i], e[i]);
+                else 
+                    Assert.AreEqual(e[i], g[i], $"Items differ at index {i}, expected {e[i]} but got {g[i]}");
+            }
         }
 
+        public void assertAllEqual(ICollection given, ICollection expected)
+        {
+            assertItemsEqual(given, expected);
+        }
+
+
         public void assertEqual(object given, object expected)
         {
+            if (given is NDArray && expected is NDArray)
+            {
+                assertItemsEqual((given as NDArray).Array, (expected as NDArray).Array);
+                return;
+            }
+            if (given is Hashtable && expected is Hashtable)
+            {
+                Assert.AreEqual(JObject.FromObject(expected).ToString(), JObject.FromObject(given).ToString());
+                return;
+            }
             if (given is ICollection && expected is ICollection)
             {
                 assertItemsEqual(given as ICollection, expected as ICollection);
@@ -54,6 +84,16 @@ public void assertIsNotNone(object given)
             Assert.IsNotNull(given);
         }
 
+        public void assertFalse(bool cond)
+        {
+            Assert.IsFalse(cond);
+        }
+
+        public void assertTrue(bool cond)
+        {
+            Assert.IsTrue(cond);
+        }
+
         #endregion
 
         #region tensor evaluation
diff --git a/test/TensorFlowNET.UnitTest/nest_test/NestTest.cs b/test/TensorFlowNET.UnitTest/nest_test/NestTest.cs
