Ptex

PtexCache.cpp
Go to the documentation of this file.
1 /*
2 PTEX SOFTWARE
3 Copyright 2014 Disney Enterprises, Inc. All rights reserved
4
5 Redistribution and use in source and binary forms, with or without
6 modification, are permitted provided that the following conditions are
7 met:
8
9 * Redistributions of source code must retain the above copyright
10 notice, this list of conditions and the following disclaimer.
11
12 * Redistributions in binary form must reproduce the above copyright
13 notice, this list of conditions and the following disclaimer in
14 the documentation and/or other materials provided with the
15 distribution.
16
17 * The names "Disney", "Walt Disney Pictures", "Walt Disney Animation
18 Studios" or the names of its contributors may NOT be used to
19 endorse or promote products derived from this software without
20 specific prior written permission from Walt Disney Pictures.
21
22 Disclaimer: THIS SOFTWARE IS PROVIDED BY WALT DISNEY PICTURES AND
23 CONTRIBUTORS "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING,
24 BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY, FITNESS
25 FOR A PARTICULAR PURPOSE, NONINFRINGEMENT AND TITLE ARE DISCLAIMED.
26 IN NO EVENT SHALL WALT DISNEY PICTURES, THE COPYRIGHT HOLDER OR
27 CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL,
28 EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO,
29 PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR
30 PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND BASED ON ANY
31 THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
32 (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
33 OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGES.
34 */
35
69 #include "PtexPlatform.h"
70 #include <algorithm>
71 #include <sys/types.h>
72 #include <sys/stat.h>
73 #include <stdlib.h>
74 #include <iostream>
75 #include <ctype.h>
76 #include "Ptexture.h"
77 #include "PtexReader.h"
78 #include "PtexCache.h"
79
80
81 PTEX_NAMESPACE_BEGIN
82
83 void PtexCachedReader::release()
84 {
85 if (0 == unref()) {
86 _cache->logRecentlyUsed(this);
87 }
88 }
89
90
91 bool PtexReaderCache::findFile(const char*& filename, std::string& buffer, Ptex::String& error)
92 {
93 bool isAbsolute = (filename[0] == '/'
94 #ifdef WINDOWS
95 || filename[0] == '\\'
96 || (isalpha(filename[0]) && filename[1] == ':')
97 #endif
98 );
99 if (isAbsolute || _searchdirs.empty()) return true; // no need to search
100
101 // file is relative, search in searchpath
102 buffer.reserve(256); // minimize reallocs (will grow automatically)
103 struct stat statbuf;
104 for (size_t i = 0, size = _searchdirs.size(); i < size; i++) {
105 buffer = _searchdirs[i];
106 buffer += "/";
107 buffer += filename;
108 if (stat(buffer.c_str(), &statbuf) == 0) {
109 filename = buffer.c_str();
110 return true;
111 }
112 }
113 // not found
114 std::string errstr = "Can't find ptex file: ";
115 errstr += filename;
116 error = errstr.c_str();
117 return false;
118 }
119
120
121 PtexTexture* PtexReaderCache::get(const char* filename, Ptex::String& error)
122 {
123 // lookup reader in map
124 StringKey key(filename);
125 PtexCachedReader* reader = _files.get(key);
126 bool isNew = false;
127
128 if (reader) {
129 if (!reader->ok()) return 0;
130 if (reader->pendingPurge()) {
131 // a previous purge attempt was made and file was busy. Try again now.
132 purge(reader);
133 }
134 reader->ref();
135 } else {
136 reader = new PtexCachedReader(_premultiply, _io, _err, this);
137 isNew = true;
138 }
139
140 bool needOpen = reader->needToOpen();
141 if (needOpen) {
142 std::string buffer;
143 const char* pathToOpen = filename;
144 // search for the file (unless we have an I/O handler)
145 if (_io || findFile(pathToOpen, buffer, error)) {
146 reader->open(pathToOpen, error);
147 } else {
148 // flag reader as invalid so we don't try to open it again on next lookup
149 reader->invalidate();
150 }
151 }
152
153 if (isNew) {
154 size_t newMemUsed = 0;
155 PtexCachedReader* newreader = reader;
156 reader = _files.tryInsert(key, reader, newMemUsed);
157 adjustMemUsed(newMemUsed);
158 if (reader != newreader) {
159 // another thread got here first
160 reader->ref();
161 delete newreader;
162 }
163 }
164
165 if (!reader->ok()) {
166 reader->unref();
167 return 0;
168 }
169
170 if (needOpen) {
171 reader->logOpen();
172 }
173
174 return reader;
175 }
176
177 PtexCache* PtexCache::create(int maxFiles, size_t maxMem, bool premultiply,
178 PtexInputHandler* inputHandler,
179 PtexErrorHandler* errorHandler)
180 {
181 // set default files to 100
182 if (maxFiles <= 0) maxFiles = 100;
183
184 return new PtexReaderCache(maxFiles, maxMem, premultiply, inputHandler, errorHandler);
185 }
186
187
188 void PtexReaderCache::logRecentlyUsed(PtexCachedReader* reader)
189 {
190 while (1) {
191 MruList* mruList = _mruList;
192 int slot = AtomicIncrement(&mruList->next)-1;
193 if (slot < numMruFiles) {
194 mruList->files[slot] = reader;
195 return;
196 }
197 // no mru slot available, process mru list and try again
198 do processMru();
199 while (_mruList->next >= numMruFiles);
200 }
201 }
202
203 void PtexReaderCache::processMru()
204 {
205 // use a non-blocking lock so we can proceed as soon as space has been freed in the mru list
206 // (which happens almost immediately in the processMru thread that has the lock)
207 if (!_mruLock.trylock()) return;
208 if (_mruList->next < numMruFiles) {
209 _mruLock.unlock();
210 return;
211 }
212
213 // switch mru buffers and reset slot counter so other threads can proceed immediately
214 MruList* mruList = _mruList;
215 AtomicStore(&_mruList, _prevMruList);
216 _prevMruList = mruList;
217
218 // extract relevant stats and add to open/active list
219 size_t memUsedChange = 0, filesOpenChange = 0;
220 for (int i = 0; i < numMruFiles; ++i) {
221 PtexCachedReader* reader;
222 do { reader = mruList->files[i]; } while (!reader); // loop on (unlikely) race condition
223 mruList->files[i] = 0;
224 memUsedChange += reader->getMemUsedChange();
225 size_t opens = reader->getOpensChange();
226 size_t blockReads = reader->getBlockReadsChange();
227 filesOpenChange += opens;
228 if (opens || blockReads) {
229 _fileOpens += opens;
230 _blockReads += blockReads;
231 _openFiles.push(reader);
232 }
233 if (_maxMem) {
234 _activeFiles.push(reader);
235 }
236 }
237 AtomicStore(&mruList->next, 0);
238 adjustMemUsed(memUsedChange);
239 adjustFilesOpen(filesOpenChange);
240
241 bool shouldPruneFiles = _filesOpen > _maxFiles;
242 bool shouldPruneData = _maxMem && _memUsed > _maxMem;
243
244 if (shouldPruneFiles) {
245 pruneFiles();
246 }
247 if (shouldPruneData) {
248 pruneData();
249 }
250 _mruLock.unlock();
251 }
252
253
254 void PtexReaderCache::pruneFiles()
255 {
256 size_t numToClose = _filesOpen - _maxFiles;
257 if (numToClose > 0) {
258 while (numToClose) {
259 PtexCachedReader* reader = _openFiles.pop();
260 if (!reader) { _filesOpen = 0; break; }
261 if (reader->tryClose()) {
262 --numToClose;
263 --_filesOpen;
264 }
265 }
266 }
267 }
268
269
270 void PtexReaderCache::pruneData()
271 {
272 size_t memUsedChangeTotal = 0;
273 size_t memUsed = _memUsed;
274 while (memUsed + memUsedChangeTotal > _maxMem) {
275 PtexCachedReader* reader = _activeFiles.pop();
276 if (!reader) break;
277 size_t memUsedChange;
278 if (reader->tryPrune(memUsedChange)) {
279 // Note: after clearing, memUsedChange is negative
280 memUsedChangeTotal += memUsedChange;
281 }
282 }
283 adjustMemUsed(memUsedChangeTotal);
284 }
285
286
287 void PtexReaderCache::purge(PtexTexture* texture)
288 {
289 PtexCachedReader* reader = static_cast<PtexCachedReader*>(texture);
290 reader->unref();
291 purge(reader);
292 reader->ref();
293 }
294
295
296 void PtexReaderCache::purge(const char* filename)
297 {
298 StringKey key(filename);
299 PtexCachedReader* reader = _files.get(key);
300 if (reader) purge(reader);
301 }
302
303 void PtexReaderCache::purge(PtexCachedReader* reader)
304 {
305 size_t memUsedChange;
306 if (reader->tryPurge(memUsedChange)) {
307 adjustMemUsed(memUsedChange);
308 }
309 }
310
311 void PtexReaderCache::Purger::operator()(PtexCachedReader* reader)
312 {
313 size_t memUsedChange;
314 if (reader->tryPurge(memUsedChange)) {
315 memUsedChangeTotal += memUsedChange;
316 }
317 }
318
319 void PtexReaderCache::purgeAll()
320 {
321 Purger purger;
322 _files.foreach(purger);
323 adjustMemUsed(purger.memUsedChangeTotal);
324 }
325
326 void PtexReaderCache::getStats(Stats& stats)
327 {
328 stats.memUsed = _memUsed;
329 stats.peakMemUsed = _peakMemUsed;
330 stats.filesOpen = _filesOpen;
331 stats.peakFilesOpen = _peakFilesOpen;
332 stats.filesAccessed = _files.size();
333 stats.fileReopens = _fileOpens < stats.filesAccessed ? 0 : _fileOpens - stats.filesAccessed;
334 stats.blockReads = _blockReads;
335 }
336
337 PTEX_NAMESPACE_END
PtexReaderCache::adjustMemUsed
void adjustMemUsed(size_t amount)
Definition: PtexCache.h:253
PtexReaderCache::MruList
Definition: PtexCache.h:292
PtexCachedReader::_cache
PtexReaderCache * _cache
Definition: PtexCache.h:110
PtexReaderCache::get
virtual PtexTexture * get(const char *path, Ptex::String &error)
Access a texture.
Definition: PtexCache.cpp:121
PtexCache::Stats::peakMemUsed
uint64_t peakMemUsed
Definition: Ptexture.h:794
Ptex::String
Memory-managed string.
Definition: Ptexture.h:308
PtexReader::logOpen
void logOpen()
Definition: PtexReader.h:72
PtexHashMap::size
uint32_t size() const
Definition: PtexHashMap.h:201
PtexReaderCache::getStats
virtual void getStats(Stats &stats)
Get stats.
Definition: PtexCache.cpp:326
PtexCache
File-handle and memory cache for reading ptex files.
Definition: Ptexture.h:697
PtexReaderCache::_err
PtexErrorHandler * _err
Definition: PtexCache.h:281
AtomicIncrement
PTEX_INLINE T AtomicIncrement(volatile T *target)
Definition: PtexPlatform.h:213
PtexReaderCache::_peakFilesOpen
size_t _peakFilesOpen
Definition: PtexCache.h:304
PtexReaderCache::_maxMem
size_t _maxMem
Definition: PtexCache.h:279
Mutex::trylock
bool trylock()
Definition: PtexPlatform.h:130
PtexReaderCache::pruneFiles
void pruneFiles()
Definition: PtexCache.cpp:254
PtexCache::Stats::memUsed
uint64_t memUsed
Definition: Ptexture.h:793
PtexReaderCache::MruList::next
volatile int next
Definition: PtexCache.h:293
PtexPlatform.h
Platform-specific classes, functions, and includes.
PtexCache.h
PtexReaderCache::findFile
bool findFile(const char *&filename, std::string &buffer, Ptex::String &error)
Definition: PtexCache.cpp:91
Ptex::String::c_str
const char * c_str() const
Definition: Ptexture.h:317
PtexReader::invalidate
void invalidate()
Definition: PtexReader.h:66
PtexCachedReader::tryPurge
bool tryPurge(size_t &memUsedChange)
Definition: PtexCache.h:162
PtexErrorHandler
Custom handler interface redirecting Ptex error messages.
Definition: Ptexture.h:671
PtexReaderCache::_searchdirs
std::vector< std::string > _searchdirs
Definition: PtexCache.h:283
PtexReaderCache::Purger::operator()
void operator()(PtexCachedReader *reader)
Definition: PtexCache.cpp:311
PtexCache::Stats::filesOpen
uint64_t filesOpen
Definition: Ptexture.h:795
PtexReaderCache::logRecentlyUsed
void logRecentlyUsed(PtexCachedReader *reader)
Definition: PtexCache.cpp:188
PtexReaderCache::numMruFiles
static const int numMruFiles
Definition: PtexCache.h:291
PtexCachedReader::release
virtual void release()
Release resources held by this pointer (pointer becomes invalid).
Definition: PtexCache.cpp:83
PtexReaderCache
Cache for reading Ptex texture files.
Definition: PtexCache.h:197
PtexReaderCache::_mruList
MruList *volatile _mruList
Definition: PtexCache.h:297
PtexReader::needToOpen
bool needToOpen() const
Definition: PtexReader.h:57
PtexReaderCache::_files
FileMap _files
Definition: PtexCache.h:285
PtexReaderCache::_mruLock
Mutex _mruLock
Definition: PtexCache.h:289
PtexLruList::pop
T * pop()
Definition: PtexCache.h:95
PtexReaderCache::_maxFiles
size_t _maxFiles
Definition: PtexCache.h:278
PtexCache::Stats::blockReads
uint64_t blockReads
Definition: Ptexture.h:799
PtexReaderCache::_openFiles
PtexLruList< PtexCachedReader, &PtexCachedReader::_openFilesItem > _openFiles
Definition: PtexCache.h:300
PtexCache::Stats
Definition: Ptexture.h:792
PtexCache::Stats::filesAccessed
uint64_t filesAccessed
Definition: Ptexture.h:797
PtexReaderCache::Purger
Definition: PtexCache.h:268
PtexReaderCache::_blockReads
size_t _blockReads
Definition: PtexCache.h:306
PtexReaderCache::_activeFiles
PtexLruList< PtexCachedReader, &PtexCachedReader::_activeFilesItem > _activeFiles
Definition: PtexCache.h:301
PtexReaderCache::_premultiply
bool _premultiply
Definition: PtexCache.h:286
PtexCachedReader::getOpensChange
size_t getOpensChange()
Definition: PtexCache.h:180
PtexReader::tryClose
bool tryClose()
Definition: PtexReader.cpp:219
PtexCachedReader::ref
void ref()
Definition: PtexCache.h:138
PtexCachedReader::unref
int32_t unref()
Definition: PtexCache.h:146
PtexLruList::push
void push(T *node)
Definition: PtexCache.h:88
PtexReaderCache::_filesOpen
volatile size_t _filesOpen
Definition: PtexCache.h:288
PtexReaderCache::processMru
void processMru()
Definition: PtexCache.cpp:203
PtexHashMap::foreach
void foreach(Fn &fn)
Definition: PtexHashMap.h:252
PtexTexture
Interface for reading data from a ptex file.
Definition: Ptexture.h:470
PtexReaderCache::_io
PtexInputHandler * _io
Definition: PtexCache.h:280
PtexCache::create
static PtexCache * create(int maxFiles, size_t maxMem, bool premultiply=false, PtexInputHandler *inputHandler=0, PtexErrorHandler *errorHandler=0)
Create a cache with the specified limits.
Definition: PtexCache.cpp:177
PtexCachedReader::tryPrune
bool tryPrune(size_t &memUsedChange)
Definition: PtexCache.h:152
PtexCachedReader::getBlockReadsChange
size_t getBlockReadsChange()
Definition: PtexCache.h:187
PtexReader::pendingPurge
bool pendingPurge() const
Definition: PtexReader.h:62
PtexCache::Stats::fileReopens
uint64_t fileReopens
Definition: Ptexture.h:798
PtexCache::Stats::peakFilesOpen
uint64_t peakFilesOpen
Definition: Ptexture.h:796
PtexReader::open
bool open(const char *path, Ptex::String &error)
Definition: PtexReader.cpp:137
PtexReaderCache::_memUsed
volatile size_t _memUsed
Definition: PtexCache.h:287
Mutex::unlock
void unlock()
Definition: PtexPlatform.h:131
PtexReaderCache::adjustFilesOpen
void adjustFilesOpen(size_t amount)
Definition: PtexCache.h:259
PtexReaderCache::Purger::memUsedChangeTotal
size_t memUsedChangeTotal
Definition: PtexCache.h:269
PtexReaderCache::purgeAll
virtual void purgeAll()
Remove all texture files from the cache.
Definition: PtexCache.cpp:319
PtexReaderCache::purge
virtual void purge(PtexTexture *)
Remove a texture file from the cache.
Definition: PtexCache.cpp:287
PtexCachedReader::getMemUsedChange
size_t getMemUsedChange()
Definition: PtexCache.h:173
PtexReaderCache::_prevMruList
MruList *volatile _prevMruList
Definition: PtexCache.h:298
PtexReader::ok
bool ok() const
Definition: PtexReader.h:64
PTEX_NAMESPACE_BEGIN
Definition: PtexSeparableKernel.cpp:42
PtexCachedReader
Definition: PtexCache.h:108
PtexReaderCache::MruList::files
PtexCachedReader *volatile files[numMruFiles]
Definition: PtexCache.h:294
AtomicStore
PTEX_INLINE void AtomicStore(T volatile *target, T value)
Definition: PtexPlatform.h:272
PtexReader.h
PtexReaderCache::_fileOpens
size_t _fileOpens
Definition: PtexCache.h:305
PTEX_NAMESPACE_END
#define PTEX_NAMESPACE_END
Definition: PtexVersion.h:62
Ptexture.h
Public API classes for reading, writing, caching, and filtering Ptex files.
PtexHashMap::tryInsert
Value tryInsert(Key &key, Value value, size_t &newMemUsed)
Definition: PtexHashMap.h:224
PtexInputHandler
Custom handler interface for intercepting and redirecting Ptex input stream calls.
Definition: Ptexture.h:632
PtexReaderCache::pruneData
void pruneData()
Definition: PtexCache.cpp:270
PtexHashMap::get
Value get(Key &key)
Definition: PtexHashMap.h:203
StringKey
Definition: PtexHashMap.h:82
PtexReaderCache::_peakMemUsed
size_t _peakMemUsed
Definition: PtexCache.h:303