fix: spark df timestamp datatype detection (#59)

Co-authored-by: Patil <pradyot.patil@ing.com>

Author: pradyot-09

histogrammar/util.py

@@ -661,7 +661,7 @@ def _is_probable_timestamp(value, DATE_LOW=5e16, DATE_HIGH=9.9e18):
 
     # timestamp is in ns since 1970, so a huge number.
     is_ts = False
-    if isinstance(value, np.number) and not np.isnan(value):
+    if isinstance(value, (np.number,float,int)) and not np.isnan(value):
         is_ts = DATE_LOW < value < DATE_HIGH
     return is_ts
 

tests/test_basic.py

@@ -18,6 +18,7 @@
 import pickle
 import sys
 import unittest
+import numpy as np
 
 from histogrammar.defs import Factory
 from histogrammar.primitives.average import Average
@@ -40,6 +41,7 @@
 
 from histogrammar import util
 from histogrammar.util import xrange, named
+from histogrammar.util import _is_probable_timestamp
 
 tolerance = 1e-12
 util.relativeTolerance = tolerance
@@ -170,6 +172,7 @@ def runTest(self):
         self.testPlotProfileErr()
         self.testPlotStack()
         self.testSparselyBin()
+        self.testIsProbableTimestamp()
         self.testCentrallyBin()
         self.testFraction()
         self.testFractionSum()
@@ -902,6 +905,16 @@ def testSparselyBinWithFloatBinWidth(self):
         self.assertEqual(len(edges), 3)
         self.assertEqual(len(entries), 2)
         self.assertEqual(list(entries), [0, 0])
+    
+    def testIsProbableTimestamp(self):
+        self.assertEqual(True,_is_probable_timestamp(np.int64(9e17)))
+        self.assertEqual(True,_is_probable_timestamp(np.double(7.9e16)))
+        self.assertEqual(True,_is_probable_timestamp(7.9e16))
+        self.assertEqual(True,_is_probable_timestamp(6e16))
+        self.assertEqual(False,_is_probable_timestamp(np.float32(4.9e16)))
+        self.assertEqual(False,_is_probable_timestamp(False))
+        self.assertEqual(False,_is_probable_timestamp("7.9e16"))
+
 
     # CentrallyBin