问在python中通过变量释放内存使用情况
EN

Stack Overflow用户

提问于 2017-04-11 14:27:03

回答 1查看 186关注 0票数 0

我目前正在尝试将一些数据存储到.h5文件中，我很快意识到这可能需要将我的数据存储到各个部分中，因为它不可能在我的ram中处理它。我一开始使用numpy.array来压缩内存使用，但这导致我花了几天时间来格式化数据。

所以我又回到了使用list，但让程序监控内存使用情况，当它超过一个指定值时，是否会以numpy格式存储一个部分-这样另一个进程就可以加载并使用它。这样做的问题是，我认为会释放内存的并不是释放内存。由于某些原因，内存是相同的，即使我重置了变量和del变量。为什么内存没有在这里释放？

import numpy as np
import os
import resource
import sys
import gc
import math
import h5py
import SecureString
import objgraph
from numpy.lib.stride_tricks import as_strided as ast

total_frames = 15
total_frames_with_deltas = total_frames*3
dim = 40
window_height = 5


def store_file(file_name,data):
    with h5py.File(file_name,'w') as f:
        f["train_input"] = np.concatenate(data,axis=1)

def load_data_overlap(saved):
    #os.chdir(numpy_train)
    print "Inside function!..."
    if saved == False:
        train_files = np.random.randint(255,size=(1,40,690,4))
        train_input_data_interweawed_normalized = []
        print "Storing train pic to numpy"
        part = 0
        for i in xrange(100000):
            print resource.getrusage(resource.RUSAGE_SELF).ru_maxrss
            if resource.getrusage(resource.RUSAGE_SELF).ru_maxrss > 2298842112/10:
                print "Max ram storing part: " + str(part) + " At entry: " + str(i)
                print "Storing Train input"
                file_name = 'train_input_'+'part_'+str(part)+'_'+str(dim)+'_'+str(total_frames_with_deltas)+'_window_height_'+str(window_height)+'.h5'
                store_file(file_name,train_input_data_interweawed_normalized)
                part = part + 1             
                del train_input_data_interweawed_normalized
                gc.collect()
                train_input_data_interweawed_normalized = []
                raw_input("something")
            for plot in train_files:
                overlaps_reshaped = np.random.randint(10,size=(45,200,5,3))
                for ind_plot in overlaps_reshaped.reshape(overlaps_reshaped.shape[1],overlaps_reshaped.shape[0],overlaps_reshaped.shape[2],overlaps_reshaped.shape[3]): 
                    ind_plot_reshaped = ind_plot.reshape(ind_plot.shape[0],1,ind_plot.shape[1],ind_plot.shape[2])
                    train_input_data_interweawed_normalized.append(ind_plot_reshaped)
    print len(train_input_data_interweawed_normalized)

    return train_input_data_interweawed_normalized_print
#------------------------------------------------------------------------------------------------------------------------------------------------------------

saved = False
train_input = load_data_overlap(saved)

输出：

.....
223662080
224772096
225882112
226996224
228106240
229216256
230326272
Max ram storing part: 0 At entry: 135
Storing Train input
something
377118720
Max ram storing part: 1 At entry: 136
Storing Train input
something
377118720
Max ram storing part: 2 At entry: 137
Storing Train input
something