10 examples of 'load dataset in python' in Python

534
def load(*, path='./dataset3'):
535
    dataset = list()
536
    for file in os.listdir(path):
537
        if os.path.splitext(file)[1] != '.json':
538
            continue
539
        with open(f'{path}/{file}', 'r') as load_file:
540
            data = json.load(load_file, object_hook=object_hook)
541
        for item in data:
542
            dataset.append(Info(item))
543
    return Dataset(dataset)

92
def load_dataset(data_file=('%s/%s' % (DATA_DIR, DATA_FILE))):
93
94
    dataset = utils_backdoor.load_dataset(data_file, keys=['X_test', 'Y_test'])
95
96
    X_test = np.array(dataset['X_test'], dtype='float32')
97
    Y_test = np.array(dataset['Y_test'], dtype='float32')
98
99
    print('X_test shape %s' % str(X_test.shape))
100
    print('Y_test shape %s' % str(Y_test.shape))
101
102
    return X_test, Y_test

170
def load_dataset():
171
    global data_path
172
    dataset = Dataset(data_path, verbose=True)
173
    dataset_size = len(dataset.samples)
174
    assert dataset_size > 0
175
    return dataset

61
def load_pandas():
62
    data = load()
63
    # pandas <= 0.12.0 fails in the to_datetime regex on Python 3
64
    index = pd.DatetimeIndex(start=data.data['date'][0].decode('utf-8'),
65
                             periods=len(data.data), format='%Y%m%d',
66
                             freq='W-SAT')
67
    dataset = pd.DataFrame(data.data['co2'], index=index, columns=['co2'])
68
    #NOTE: this is how I got the missing values in co2.csv
69
    #new_index = pd.DatetimeIndex(start='1958-3-29', end=index[-1],
70
    #                             freq='W-SAT')
71
    #data.data = dataset.reindex(new_index)
72
    data.data = dataset
73
    return data

35
def load_dataset(data_arg, wvpath, embedding_size):
36
	wv = wordvec_class(wvpath)
37
	dm = data_class(**data_arg)
38
	return dm, wv.load_matrix(embedding_size, dm.vocab_list)

43
def load(self, *args, **kwargs):
44
    """Load data and check the channel number `c_dim`.
45
    """
46
    self._load(*args, **kwargs)
47
    click.secho(" [*] Dataset '%s' loaded." % self.name, fg="green")

22
def get_dataset():
23
    dataset_filename = 'dataset/annoted_dataset.pkl'
24
    if not os.path.isfile(dataset_filename):
25
        annoted_data = read_json_formatted()
26
        dataset = []
27
        for row in annoted_data:
28
            sources = [s.lower() for s in row['target']]
29
            targets = [s.lower() for s in row['polarity']]
30
            sentence_meta = {}
31
            sentence = row['sentence']
32
            for source, target in zip(sources, targets):
33
                sentence_meta[source] = target
34
            dataset.append({'sentence': sentence, 'meta': sentence_meta})
35
        pd.to_pickle(dataset, dataset_filename)
36
    else:
37
        dataset = pd.read_pickle(dataset_filename)
38
    return dataset

581
def load_data(self,param,dates):
582
    if param=='PM2.5':
583
        df = self.load_aqs_pm25_data(dates)
584
    elif param == 'PM10':
585
        df = self.load_aqs_pm10_data(dates)
586
    elif param == 'SPEC':
587
        df = self.load_aqs_spec_data(dates)
588
    elif param == 'CO':
589
        df = self.load_aqs_co_data(dates)
590
    elif param == 'OZONE':
591
        df = self.load_aqs_ozone_data(dates)
592
    elif param == 'SO2':
593
        df = self.load_aqs_so2_data(dates)
594
    elif param == 'VOC':
595
        df = self.load_aqs_voc_data(dates)
596
    elif param == 'NONOXNOY':
597
        df = self.load_aqs_nonoxnoy_data(dates)
598
    elif param == 'WIND':
599
        df = self.load_aqs_wind_data(dates)
600
    elif param == 'TEMP':
601
        df = self.load_aqs_temp_data(dates)
602
    elif param == 'RHDP':
603
        df = self.load_aqs_rhdp_data(dates)
604
    return df

66
def load(self):
67
    self.open_file(self.path)

15
def load_data(dataset=""):
16
    base_dir = os.path.join("data", dataset)
17
    assert os.path.exists(base_dir), \
18
        "Could not find data directory: " + base_dir
19
20
    model_path = os.path.join(base_dir, "model.pkl.gz")
21
    model = None
22
    if os.path.exists(model_path):
23
        with gzip.open(model_path, "r") as f:
24
            model = cPickle.load(f)
25
26
    train_path = os.path.join(base_dir, "train.pkl.gz")
27
    with gzip.open(train_path, "r") as f:
28
        train = cPickle.load(f)
29
30
    test_path = os.path.join(base_dir, "test.pkl.gz")
31
    with gzip.open(test_path, "r") as f:
32
        test = cPickle.load(f)
33
34
    return train, test, model