#pragma once #include "../taskflow.hpp" #include #include namespace tf { enum StorageLevel { MEMORY = 0, MEMORY_AND_DISK = 1 }; /** @class Tensor @brief a tensor contains arithmetic data in N dimensions */ template class Tensor { template friend class TensorNode; template friend class TensorExpr; template friend class TensorFrame; struct Chunk { std::vector data; std::string location; }; public: Tensor(const Tensor& tensor) = delete; Tensor(Tensor&& tensor) = delete; Tensor(std::vector shape); Tensor(std::vector shape, size_t max_chunk_size); const std::vector& shape() const; const std::vector& chunk_shape() const; size_t size() const; size_t rank() const; size_t chunk_size() const; size_t num_chunks() const; StorageLevel storage_level() const; void dump(std::ostream& ostream) const; template size_t flat_chunk_index(Is... indices) const; template size_t flat_index(Is... indices) const; private: StorageLevel _storage_level; std::vector _shape; std::vector _chunk_shape; std::vector _chunk_grid; std::vector _chunks; void _make_chunks(size_t = 65536*1024); // 65MB per chunk size_t _flat_chunk_index(size_t&, size_t) const; template size_t _flat_chunk_index(size_t&, size_t, Is...) const; size_t _flat_index(size_t&, size_t) const; template size_t _flat_index(size_t&, size_t, Is...) const; }; template Tensor::Tensor(std::vector shape) : _shape {std::move(shape)}, _chunk_shape (_shape.size()), _chunk_grid (_shape.size()) { _make_chunks(); } template Tensor::Tensor(std::vector shape, size_t max_chunk_size) : _shape {std::move(shape)}, _chunk_shape (_shape.size()), _chunk_grid (_shape.size()) { _make_chunks(std::max(1ul, max_chunk_size)); } template size_t Tensor::size() const { return std::accumulate( _shape.begin(), _shape.end(), 1, std::multiplies() ); } template size_t Tensor::num_chunks() const { return _chunks.size(); } template size_t Tensor::chunk_size() const { return _chunks[0].data.size(); } template size_t Tensor::rank() const { return _shape.size(); } template const std::vector& Tensor::shape() const { return _shape; } template const std::vector& Tensor::chunk_shape() const { return _chunk_shape; } template template size_t Tensor::flat_chunk_index(Is... rest) const { if(sizeof...(Is) != rank()) { TF_THROW("index rank dose not match tensor rank"); } size_t offset; return _flat_chunk_index(offset, rest...); } template size_t Tensor::_flat_chunk_index(size_t& offset, size_t id) const { offset = 1; return id/_chunk_shape.back(); } template template size_t Tensor::_flat_chunk_index( size_t& offset, size_t id, Is... rest ) const { auto i = _flat_chunk_index(offset, rest...); offset *= _chunk_grid[_chunk_shape.size() - (sizeof...(Is))]; return (id/_chunk_shape[_chunk_shape.size() - sizeof...(Is) - 1])*offset + i; } template template size_t Tensor::flat_index(Is... rest) const { if(sizeof...(Is) != rank()) { TF_THROW("index rank dose not match tensor rank"); } size_t offset; return _flat_index(offset, rest...); } template size_t Tensor::_flat_index(size_t& offset, size_t id) const { offset = 1; return id; } template template size_t Tensor::_flat_index(size_t& offset, size_t id, Is... rest) const { auto i = _flat_index(offset, rest...); offset *= _shape[_shape.size() - (sizeof...(Is))]; return id*offset + i; } template void Tensor::dump(std::ostream& os) const { os << "Tensor<" << typeid(T).name() << "> {\n" << " shape=["; for(size_t i=0; i<_shape.size(); ++i) { if(i) os << 'x'; os << _shape[i]; } os << "], chunk=["; for(size_t i=0; i<_chunk_shape.size(); ++i) { if(i) os << 'x'; os << _chunk_shape[i]; } os << "], pgrid=["; for(size_t i=0; i<_chunk_grid.size(); ++i) { if(i) os << 'x'; os << _chunk_grid[i]; } os << "]\n}\n"; } template void Tensor::_make_chunks(size_t M) { size_t P = 1; size_t N = 1; for(int i=_shape.size()-1; i>=0; i--) { if(M >= _shape[i]) { _chunk_shape[i] = _shape[i]; _chunk_grid[i] = 1; N *= _chunk_shape[i]; M /= _shape[i]; } else { _chunk_shape[i] = M; _chunk_grid[i] = (_shape[i] + _chunk_shape[i] - 1) / _chunk_shape[i]; P *= _chunk_grid[i]; N *= _chunk_shape[i]; for(i--; i>=0; i--) { _chunk_shape[i] = 1; _chunk_grid[i] = _shape[i]; P *= _chunk_grid[i]; } break; } } _chunks.resize(P); // we allocate the first data in memory _chunks[0].data.resize(N); // TODO: the rest sits in the disk for(size_t i=1; i<_chunks.size(); ++i) { } // assign the storage level _storage_level = (_chunks.size() <= 1) ? MEMORY : MEMORY_AND_DISK; } } // end of namespace tf -----------------------------------------------------