test

GenDs.py

@@ -0,0 +1,66 @@
+# get data
+import pandas as pd
+import numpy as np
+import tensorflow as tf
+raw_data = pd.read_pickle("./data/raw_feature.pkl")
+X_train = pd.read_pickle("./data/X_train.pkl")
+y_train = pd.read_pickle("./data/y_train.pkl")
+X_test = pd.read_pickle("./data/X_test.pkl")
+y_test = pd.read_pickle("./data/y_test.pkl")
+
+# preprocess data
+from sklearn.preprocessing import StandardScaler,MinMaxScaler
+std = StandardScaler()
+train_shape = y_train.shape
+test_shape = y_test.shape
+y_train = std.fit_transform(np.reshape(y_train,(-1,y_train.shape[-2]*y_train.shape[-1])))
+y_train = y_train.reshape(train_shape)
+y_test = std.transform(np.reshape(y_test,(-1,y_test.shape[-2]*y_test.shape[-1])))
+y_test = y_test.reshape(test_shape)
+
+# gen dataset
+from numpy.lib.stride_tricks import as_strided
+
+def create_rolling_window(matrix,t):
+    matrix_shape = matrix.shape
+    return_length = matrix_shape[0] - t 
+    dataset = tf.data.Dataset.from_tensor_slices(matrix)
+    windows = dataset.window(t,shift = 1,drop_remainder=True)
+    windows = windows.take(return_length)
+    windows = windows.flat_map(lambda window: window.batch(t))
+
+
+    return windows
+def create_result_ds(matrix,delay):
+    dataset = tf.data.Dataset.from_tensor_slices(matrix)
+    dataset = dataset.skip(delay)
+    return dataset
+
+def combine_ds(ds1,ds2):
+    combined_ds = tf.data.Dataset.zip(((ds1),ds2))
+    combined_ds = combined_ds.batch(batch_size=32)
+    return combined_ds
+
+def gen_train_ds(X_train,y_train,step_len):
+    X = create_rolling_window(y_train,step_len)
+    y = create_result_ds(y_train,step_len)
+    train_ds = combine_ds(X,y)
+    return train_ds
+
+def gen_test_ds(X_test,y_test,step_len):
+    X = create_rolling_window(y_test,step_len)
+    y = create_result_ds(y_test,step_len)
+    test_ds = combine_ds(X,y)
+    return test_ds
+# gen dataset
+time_step = 48
+train_ds = gen_train_ds(X_train,y_train,step_len=time_step)
+test_ds = gen_test_ds(X_test,y_test,step_len=time_step)
+
+class GenDs(object):
+    def gen_train(time_step = 48):
+        train_ds = gen_train_ds(X_train,y_train,step_len=time_step)
+        return train_ds,std
+    def gen_test(time_step = 48):
+        test_ds = gen_test_ds(X_test,y_test,step_len=time_step)
+        return test_ds

ModelPerformance.py

@@ -0,0 +1,38 @@
+import numpy as np
+import  matplotlib.pyplot as plt
+from sklearn.metrics import mean_squared_error
+def compare_result(predict,real):
+    assert(predict.shape == real.shape)
+    original_shape = predict.shape
+    predict_flatten = np.reshape(predict,(-1,original_shape[-2]*original_shape[-1]))
+    real_flatten = np.reshape(real, (-1,original_shape[-2]*original_shape[-1]))
+
+
+    # assert(np.array_equal(predict_flatten.reshape(original_shape), predict))
+
+    print(mean_squared_error(predict_flatten,real_flatten))
+def plot_result(predict,real,batch =0,in_out = 0):
+
+    vmin = min(predict[batch].min(), real[batch].min())
+    vmax = max(predict[batch].max(), real[batch].max())
+
+    cmap = 'viridis'
+    fig, axs = plt.subplots(1, 3)
+
+    axs[0].set_title("predict")
+    axs[0].imshow(predict[batch,in_out,:,:],cmap = cmap,vmin = vmin,vmax = vmax)
+    axs[1].set_title("real")
+    axs[1].imshow(real[batch,in_out,:,:],cmap = cmap,vmin = vmin,vmax = vmax)
+    axs[2].set_title("diff")
+    axs[2].imshow(np.abs(predict[batch,in_out,:,:] - real[batch,in_out,:,:]), cmap = cmap,vmin = vmin,vmax = vmax)
+
+    plt.show()
+def std_inverse(std,data):
+    original_shape = data.shape
+    try:
+        data = data.copy()
+    except Exception:
+        pass
+    inversed_data = std.inverse_transform(np.reshape(data,(-1,data.shape[-2]*data.shape[-1])))
+    inversed_data = inversed_data.reshape(original_shape)
+    return inversed_data

gen_dataset.ipynb

@@ -0,0 +1,209 @@
+{
+ "cells": [
+  {
+   "cell_type": "code",
+   "execution_count": 10,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "# get data\n",
+    "import pandas as pd\n",
+    "import numpy as np\n",
+    "import tensorflow as tf\n",
+    "raw_data = pd.read_pickle(\"./data/raw_feature.pkl\")\n",
+    "X_train = pd.read_pickle(\"./data/X_train.pkl\")\n",
+    "y_train = pd.read_pickle(\"./data/y_train.pkl\")\n",
+    "X_test = pd.read_pickle(\"./data/X_test.pkl\")\n",
+    "y_test = pd.read_pickle(\"./data/y_test.pkl\")\n",
+    "\n",
+    "\n",
+    "trip_avg_in = y_train.mean(axis = 0)[0]\n",
+    "trip_avg_out = y_train.mean(axis=0)[1]"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 11,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "# preprocess data\n",
+    "from sklearn.preprocessing import StandardScaler,MinMaxScaler\n",
+    "std = StandardScaler()\n",
+    "train_shape = y_train.shape\n",
+    "test_shape = y_test.shape\n",
+    "y_train = std.fit_transform(np.reshape(y_train,(-1,y_train.shape[-2]*y_train.shape[-1])))\n",
+    "y_train = y_train.reshape(train_shape)\n",
+    "y_test = std.transform(np.reshape(y_test,(-1,y_test.shape[-2]*y_test.shape[-1])))\n",
+    "y_test = y_test.reshape(test_shape)\n",
+    "\n"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 12,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "# gen dataset\n",
+    "from numpy.lib.stride_tricks import as_strided\n",
+    "\n",
+    "def create_rolling_window(matrix,t):\n",
+    "    matrix_shape = matrix.shape\n",
+    "    return_length = matrix_shape[0] - t \n",
+    "    dataset = tf.data.Dataset.from_tensor_slices(matrix)\n",
+    "    windows = dataset.window(t,shift = 1,drop_remainder=True)\n",
+    "    windows = windows.take(return_length)\n",
+    "    windows = windows.flat_map(lambda window: window.batch(t))\n",
+    "\n",
+    "    \n",
+    "    return windows\n",
+    "def create_result_ds(matrix,delay):\n",
+    "    dataset = tf.data.Dataset.from_tensor_slices(matrix)\n",
+    "    dataset = dataset.skip(delay)\n",
+    "    return dataset\n",
+    "\n",
+    "def combine_ds(ds1,ds2):\n",
+    "    combined_ds = tf.data.Dataset.zip(((ds1),ds2))\n",
+    "    combined_ds = combined_ds.batch(batch_size=32)\n",
+    "    return combined_ds\n",
+    "\n",
+    "def gen_train_ds(X_train,y_train,step_len):\n",
+    "    X = create_rolling_window(y_train,step_len)\n",
+    "    y = create_result_ds(y_train,step_len)\n",
+    "    train_ds = combine_ds(X,y)\n",
+    "    return train_ds\n",
+    "\n",
+    "def gen_test_ds(X_test,y_test,step_len):\n",
+    "    X = create_rolling_window(y_test,step_len)\n",
+    "    y = create_result_ds(y_test,step_len)\n",
+    "    test_ds = combine_ds(X,y)\n",
+    "    return test_ds"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 13,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "# gen dataset\n",
+    "time_step = 48\n",
+    "train_ds = gen_train_ds(X_train,y_train,step_len=time_step)\n",
+    "test_ds = gen_test_ds(X_test,y_test,step_len=time_step)"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 14,
+   "metadata": {},
+   "outputs": [
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "(32, 48, 2, 16, 8) (32, 2, 16, 8)\n",
+      "(32, 48, 2, 16, 8) (32, 2, 16, 8)\n",
+      "(32, 48, 2, 16, 8) (32, 2, 16, 8)\n",
+      "(32, 48, 2, 16, 8) (32, 2, 16, 8)\n",
+      "(32, 48, 2, 16, 8) (32, 2, 16, 8)\n",
+      "(32, 48, 2, 16, 8) (32, 2, 16, 8)\n",
+      "(32, 48, 2, 16, 8) (32, 2, 16, 8)\n",
+      "(32, 48, 2, 16, 8) (32, 2, 16, 8)\n",
+      "(32, 48, 2, 16, 8) (32, 2, 16, 8)\n",
+      "(32, 48, 2, 16, 8) (32, 2, 16, 8)\n",
+      "(32, 48, 2, 16, 8) (32, 2, 16, 8)\n",
+      "(32, 48, 2, 16, 8) (32, 2, 16, 8)\n",
+      "(32, 48, 2, 16, 8) (32, 2, 16, 8)\n",
+      "(32, 48, 2, 16, 8) (32, 2, 16, 8)\n",
+      "(32, 48, 2, 16, 8) (32, 2, 16, 8)\n",
+      "(32, 48, 2, 16, 8) (32, 2, 16, 8)\n",
+      "(32, 48, 2, 16, 8) (32, 2, 16, 8)\n",
+      "(32, 48, 2, 16, 8) (32, 2, 16, 8)\n",
+      "(32, 48, 2, 16, 8) (32, 2, 16, 8)\n",
+      "(32, 48, 2, 16, 8) (32, 2, 16, 8)\n",
+      "(32, 48, 2, 16, 8) (32, 2, 16, 8)\n",
+      "(32, 48, 2, 16, 8) (32, 2, 16, 8)\n",
+      "(32, 48, 2, 16, 8) (32, 2, 16, 8)\n",
+      "(32, 48, 2, 16, 8) (32, 2, 16, 8)\n",
+      "(32, 48, 2, 16, 8) (32, 2, 16, 8)\n",
+      "(32, 48, 2, 16, 8) (32, 2, 16, 8)\n",
+      "(32, 48, 2, 16, 8) (32, 2, 16, 8)\n",
+      "(32, 48, 2, 16, 8) (32, 2, 16, 8)\n",
+      "(32, 48, 2, 16, 8) (32, 2, 16, 8)\n",
+      "(32, 48, 2, 16, 8) (32, 2, 16, 8)\n",
+      "(32, 48, 2, 16, 8) (32, 2, 16, 8)\n",
+      "(32, 48, 2, 16, 8) (32, 2, 16, 8)\n",
+      "(32, 48, 2, 16, 8) (32, 2, 16, 8)\n",
+      "(32, 48, 2, 16, 8) (32, 2, 16, 8)\n",
+      "(32, 48, 2, 16, 8) (32, 2, 16, 8)\n",
+      "(32, 48, 2, 16, 8) (32, 2, 16, 8)\n",
+      "(32, 48, 2, 16, 8) (32, 2, 16, 8)\n",
+      "(16, 48, 2, 16, 8) (16, 2, 16, 8)\n",
+      "38\n"
+     ]
+    }
+   ],
+   "source": [
+    "count = 0\n",
+    "for X,y in train_ds:\n",
+    "    print (X.shape,y.shape)\n",
+    "    count+=1\n",
+    "print(count)\n",
+    "\n"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 15,
+   "metadata": {},
+   "outputs": [
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "(32, 48, 2, 16, 8) (32, 2, 16, 8)\n",
+      "(32, 48, 2, 16, 8) (32, 2, 16, 8)\n",
+      "(32, 48, 2, 16, 8) (32, 2, 16, 8)\n",
+      "(32, 48, 2, 16, 8) (32, 2, 16, 8)\n",
+      "(32, 48, 2, 16, 8) (32, 2, 16, 8)\n",
+      "(32, 48, 2, 16, 8) (32, 2, 16, 8)\n"
+     ]
+    }
+   ],
+   "source": [
+    "\n",
+    "for X,y in test_ds:\n",
+    "    print(X.shape,y.shape)"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {},
+   "outputs": [],
+   "source": []
+  }
+ ],
+ "metadata": {
+  "kernelspec": {
+   "display_name": "Python 3",
+   "language": "python",
+   "name": "python3"
+  },
+  "language_info": {
+   "codemirror_mode": {
+    "name": "ipython",
+    "version": 3
+   },
+   "file_extension": ".py",
+   "mimetype": "text/x-python",
+   "name": "python",
+   "nbconvert_exporter": "python",
+   "pygments_lexer": "ipython3",
+   "version": "3.11.2"
+  }
+ },
+ "nbformat": 4,
+ "nbformat_minor": 2
+}