Ptex
PtexCache.cpp
Go to the documentation of this file.
1 /*
2 PTEX SOFTWARE
3 Copyright 2014 Disney Enterprises, Inc. All rights reserved
4 
5 Redistribution and use in source and binary forms, with or without
6 modification, are permitted provided that the following conditions are
7 met:
8 
9  * Redistributions of source code must retain the above copyright
10  notice, this list of conditions and the following disclaimer.
11 
12  * Redistributions in binary form must reproduce the above copyright
13  notice, this list of conditions and the following disclaimer in
14  the documentation and/or other materials provided with the
15  distribution.
16 
17  * The names "Disney", "Walt Disney Pictures", "Walt Disney Animation
18  Studios" or the names of its contributors may NOT be used to
19  endorse or promote products derived from this software without
20  specific prior written permission from Walt Disney Pictures.
21 
22 Disclaimer: THIS SOFTWARE IS PROVIDED BY WALT DISNEY PICTURES AND
23 CONTRIBUTORS "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING,
24 BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY, FITNESS
25 FOR A PARTICULAR PURPOSE, NONINFRINGEMENT AND TITLE ARE DISCLAIMED.
26 IN NO EVENT SHALL WALT DISNEY PICTURES, THE COPYRIGHT HOLDER OR
27 CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL,
28 EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO,
29 PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR
30 PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND BASED ON ANY
31 THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
32 (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
33 OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGES.
34 */
35 
69 #include "PtexPlatform.h"
70 #include <algorithm>
71 #include <sys/types.h>
72 #include <sys/stat.h>
73 #include <stdlib.h>
74 #include <iostream>
75 #include <ctype.h>
76 #include "Ptexture.h"
77 #include "PtexReader.h"
78 #include "PtexCache.h"
79 
80 
82 
84 {
85  if (0 == unref()) {
86  _cache->logRecentlyUsed(this);
87  }
88 }
89 
90 
91 bool PtexReaderCache::findFile(const char*& filename, std::string& buffer, Ptex::String& error)
92 {
93  bool isAbsolute = (filename[0] == '/'
94 #ifdef WINDOWS
95  || filename[0] == '\\'
96  || (isalpha(filename[0]) && filename[1] == ':')
97 #endif
98  );
99  if (isAbsolute || _searchdirs.empty()) return true; // no need to search
100 
101  // file is relative, search in searchpath
102  buffer.reserve(256); // minimize reallocs (will grow automatically)
103  struct stat statbuf;
104  for (size_t i = 0, size = _searchdirs.size(); i < size; i++) {
105  buffer = _searchdirs[i];
106  buffer += "/";
107  buffer += filename;
108  if (stat(buffer.c_str(), &statbuf) == 0) {
109  filename = buffer.c_str();
110  return true;
111  }
112  }
113  // not found
114  std::string errstr = "Can't find ptex file: ";
115  errstr += filename;
116  error = errstr.c_str();
117  return false;
118 }
119 
120 
121 PtexTexture* PtexReaderCache::get(const char* filename, Ptex::String& error)
122 {
123  // lookup reader in map
124  StringKey key(filename);
125  PtexCachedReader* reader = _files.get(key);
126  bool isNew = false;
127 
128  if (reader) {
129  if (!reader->ok()) return 0;
130  if (reader->pendingPurge()) {
131  // a previous purge attempt was made and file was busy. Try again now.
132  purge(reader);
133  }
134  reader->ref();
135  } else {
136  reader = new PtexCachedReader(_premultiply, _io, _err, this);
137  isNew = true;
138  }
139 
140  bool needOpen = reader->needToOpen();
141  if (needOpen) {
142  std::string buffer;
143  const char* pathToOpen = filename;
144  // search for the file (unless we have an I/O handler)
145  if (_io || findFile(pathToOpen, buffer, error)) {
146  reader->open(pathToOpen, error);
147  } else {
148  // flag reader as invalid so we don't try to open it again on next lookup
149  reader->invalidate();
150  }
151  }
152 
153  if (isNew) {
154  size_t newMemUsed = 0;
155  PtexCachedReader* newreader = reader;
156  reader = _files.tryInsert(key, reader, newMemUsed);
157  adjustMemUsed(newMemUsed);
158  if (reader != newreader) {
159  // another thread got here first
160  reader->ref();
161  delete newreader;
162  }
163  }
164 
165  if (!reader->ok()) {
166  reader->unref();
167  return 0;
168  }
169 
170  if (needOpen) {
171  reader->logOpen();
172  }
173 
174  return reader;
175 }
176 
177 PtexCache* PtexCache::create(int maxFiles, size_t maxMem, bool premultiply,
178  PtexInputHandler* inputHandler,
179  PtexErrorHandler* errorHandler)
180 {
181  // set default files to 100
182  if (maxFiles <= 0) maxFiles = 100;
183 
184  return new PtexReaderCache(maxFiles, maxMem, premultiply, inputHandler, errorHandler);
185 }
186 
187 
189 {
190  while (1) {
191  MruList* mruList = _mruList;
192  int slot = AtomicIncrement(&mruList->next)-1;
193  if (slot < numMruFiles) {
194  mruList->files[slot] = reader;
195  return;
196  }
197  // no mru slot available, process mru list and try again
198  do processMru();
199  while (_mruList->next >= numMruFiles);
200  }
201 }
202 
204 {
205  // use a non-blocking lock so we can proceed as soon as space has been freed in the mru list
206  // (which happens almost immediately in the processMru thread that has the lock)
207  if (!_mruLock.trylock()) return;
208  if (_mruList->next < numMruFiles) {
209  _mruLock.unlock();
210  return;
211  }
212 
213  // switch mru buffers and reset slot counter so other threads can proceed immediately
214  MruList* mruList = _mruList;
216  _prevMruList = mruList;
217 
218  // extract relevant stats and add to open/active list
219  size_t memUsedChange = 0, filesOpenChange = 0;
220  for (int i = 0; i < numMruFiles; ++i) {
221  PtexCachedReader* reader;
222  do { reader = mruList->files[i]; } while (!reader); // loop on (unlikely) race condition
223  mruList->files[i] = 0;
224  memUsedChange += reader->getMemUsedChange();
225  size_t opens = reader->getOpensChange();
226  size_t blockReads = reader->getBlockReadsChange();
227  filesOpenChange += opens;
228  if (opens || blockReads) {
229  _fileOpens += opens;
230  _blockReads += blockReads;
231  _openFiles.push(reader);
232  }
233  if (_maxMem) {
234  _activeFiles.push(reader);
235  }
236  }
237  AtomicStore(&mruList->next, 0);
238  adjustMemUsed(memUsedChange);
239  adjustFilesOpen(filesOpenChange);
240 
241  bool shouldPruneFiles = _filesOpen > _maxFiles;
242  bool shouldPruneData = _maxMem && _memUsed > _maxMem;
243 
244  if (shouldPruneFiles) {
245  pruneFiles();
246  }
247  if (shouldPruneData) {
248  pruneData();
249  }
250  _mruLock.unlock();
251 }
252 
253 
255 {
256  size_t numToClose = _filesOpen - _maxFiles;
257  if (numToClose > 0) {
258  while (numToClose) {
259  PtexCachedReader* reader = _openFiles.pop();
260  if (!reader) { _filesOpen = 0; break; }
261  if (reader->tryClose()) {
262  --numToClose;
263  --_filesOpen;
264  }
265  }
266  }
267 }
268 
269 
271 {
272  size_t memUsedChangeTotal = 0;
273  size_t memUsed = _memUsed;
274  while (memUsed + memUsedChangeTotal > _maxMem) {
275  PtexCachedReader* reader = _activeFiles.pop();
276  if (!reader) break;
277  size_t memUsedChange;
278  if (reader->tryPrune(memUsedChange)) {
279  // Note: after clearing, memUsedChange is negative
280  memUsedChangeTotal += memUsedChange;
281  }
282  }
283  adjustMemUsed(memUsedChangeTotal);
284 }
285 
286 
288 {
289  PtexCachedReader* reader = static_cast<PtexCachedReader*>(texture);
290  reader->unref();
291  purge(reader);
292  reader->ref();
293 }
294 
295 
296 void PtexReaderCache::purge(const char* filename)
297 {
298  StringKey key(filename);
299  PtexCachedReader* reader = _files.get(key);
300  if (reader) purge(reader);
301 }
302 
304 {
305  size_t memUsedChange;
306  if (reader->tryPurge(memUsedChange)) {
307  adjustMemUsed(memUsedChange);
308  }
309 }
310 
312 {
313  size_t memUsedChange;
314  if (reader->tryPurge(memUsedChange)) {
315  memUsedChangeTotal += memUsedChange;
316  }
317 }
318 
320 {
321  Purger purger;
322  _files.foreach(purger);
324 }
325 
327 {
328  stats.memUsed = _memUsed;
329  stats.peakMemUsed = _peakMemUsed;
330  stats.filesOpen = _filesOpen;
332  stats.filesAccessed = _files.size();
333  stats.fileReopens = _fileOpens < stats.filesAccessed ? 0 : _fileOpens - stats.filesAccessed;
334  stats.blockReads = _blockReads;
335 }
336 
PtexHashMap::tryInsert
Value tryInsert(Key &key, Value value, size_t &newMemUsed)
Definition: PtexHashMap.h:224
PtexCache::Stats::peakFilesOpen
uint64_t peakFilesOpen
Definition: Ptexture.h:796
PtexCache::create
static PtexCache * create(int maxFiles, size_t maxMem, bool premultiply=false, PtexInputHandler *inputHandler=0, PtexErrorHandler *errorHandler=0)
Create a cache with the specified limits.
Definition: PtexCache.cpp:177
PtexReaderCache::getStats
virtual void getStats(Stats &stats)
Get stats.
Definition: PtexCache.cpp:326
PtexReaderCache::_mruLock
Mutex _mruLock
Definition: PtexCache.h:289
PtexReaderCache::adjustMemUsed
void adjustMemUsed(size_t amount)
Definition: PtexCache.h:253
PtexReaderCache::_activeFiles
PtexLruList< PtexCachedReader, &PtexCachedReader::_activeFilesItem > _activeFiles
Definition: PtexCache.h:301
PTEX_NAMESPACE_END
#define PTEX_NAMESPACE_END
Definition: PtexVersion.h:62
PtexCache::Stats::peakMemUsed
uint64_t peakMemUsed
Definition: Ptexture.h:794
PtexHashMap::get
Value get(Key &key)
Definition: PtexHashMap.h:203
PtexCachedReader::getOpensChange
size_t getOpensChange()
Definition: PtexCache.h:180
PtexCache
File-handle and memory cache for reading ptex files.
Definition: Ptexture.h:697
PtexReaderCache::MruList::files
PtexCachedReader *volatile files[numMruFiles]
Definition: PtexCache.h:294
PtexReaderCache::_openFiles
PtexLruList< PtexCachedReader, &PtexCachedReader::_openFilesItem > _openFiles
Definition: PtexCache.h:300
PtexCachedReader::release
virtual void release()
Release resources held by this pointer (pointer becomes invalid).
Definition: PtexCache.cpp:83
PtexCache::Stats::blockReads
uint64_t blockReads
Definition: Ptexture.h:799
PtexReader::needToOpen
bool needToOpen() const
Definition: PtexReader.h:57
PtexCache::Stats::filesAccessed
uint64_t filesAccessed
Definition: Ptexture.h:797
PtexCachedReader::tryPrune
bool tryPrune(size_t &memUsedChange)
Definition: PtexCache.h:152
PtexReaderCache::pruneData
void pruneData()
Definition: PtexCache.cpp:270
PtexReader::tryClose
bool tryClose()
Definition: PtexReader.cpp:219
PtexHashMap::size
uint32_t size() const
Definition: PtexHashMap.h:201
PtexReaderCache::_files
FileMap _files
Definition: PtexCache.h:285
PtexReaderCache::purgeAll
virtual void purgeAll()
Remove all texture files from the cache.
Definition: PtexCache.cpp:319
PtexSeparableKernel
Definition: PtexSeparableKernel.h:48
PtexReaderCache::MruList::next
volatile int next
Definition: PtexCache.h:293
PtexReaderCache::_io
PtexInputHandler * _io
Definition: PtexCache.h:280
PtexReaderCache::_maxFiles
size_t _maxFiles
Definition: PtexCache.h:278
PtexErrorHandler
Custom handler interface redirecting Ptex error messages.
Definition: Ptexture.h:671
PtexReaderCache::_prevMruList
MruList *volatile _prevMruList
Definition: PtexCache.h:298
PtexCachedReader::getMemUsedChange
size_t getMemUsedChange()
Definition: PtexCache.h:173
PtexCache::Stats::memUsed
uint64_t memUsed
Definition: Ptexture.h:793
find_package
find_package(Doxygen) if(DOXYGEN_FOUND) add_custom_command(OUTPUT $
Definition: doc/CMakeLists.txt:1
PtexCachedReader::getBlockReadsChange
size_t getBlockReadsChange()
Definition: PtexCache.h:187
if
if(WIN32) add_definitions(/DPTEX_EXPORTS) endif(WIN32) configure_file(PtexVersion.h.in $
Definition: ptex/CMakeLists.txt:1
PtexReaderCache::pruneFiles
void pruneFiles()
Definition: PtexCache.cpp:254
PtexHalf
Half-precision (16-bit) floating-point type.
Definition: PtexHalf.h:88
PtexReaderCache::Purger::operator()
void operator()(PtexCachedReader *reader)
Definition: PtexCache.cpp:311
PtexReader::pendingPurge
bool pendingPurge() const
Definition: PtexReader.h:62
PtexReaderCache::_maxMem
size_t _maxMem
Definition: PtexCache.h:279
PtexCachedReader::ref
void ref()
Definition: PtexCache.h:138
PtexReaderCache::_premultiply
bool _premultiply
Definition: PtexCache.h:286
PtexCachedReader::unref
int32_t unref()
Definition: PtexCache.h:146
PtexTriangleKernel
Triangle filter kernel (in normalized triangle coords)
Definition: PtexTriangleKernel.h:82
AtomicStore
PTEX_INLINE void AtomicStore(T volatile *target, T value)
Definition: PtexPlatform.h:273
PtexCachedReader::_cache
PtexReaderCache * _cache
Definition: PtexCache.h:110
Mutex::trylock
bool trylock()
Definition: PtexPlatform.h:131
PtexReaderCache
Cache for reading Ptex texture files.
Definition: PtexCache.h:198
PtexInputHandler
Custom handler interface for intercepting and redirecting Ptex input stream calls.
Definition: Ptexture.h:632
PtexReader::open
bool open(const char *path, Ptex::String &error)
Definition: PtexReader.cpp:137
PTEX_NAMESPACE_BEGIN
Definition: PtexSeparableKernel.cpp:42
PtexHashMap
Definition: PtexHashMap.h:152
PtexUtils
Definition: PtexUtils.cpp:139
PtexReaderCache::get
virtual PtexTexture * get(const char *path, Ptex::String &error)
Access a texture.
Definition: PtexCache.cpp:121
PtexReaderCache::_searchdirs
std::vector< std::string > _searchdirs
Definition: PtexCache.h:283
PtexTriangleFilter
Definition: PtexTriangleFilter.h:47
PtexSeparableFilter
Definition: PtexSeparableFilter.h:46
PtexReader::ok
bool ok() const
Definition: PtexReader.h:64
PtexLruList::pop
T * pop()
Definition: PtexCache.h:95
PtexWriter
Interface for writing data to a ptex file.
Definition: Ptexture.h:823
Ptex::String
Memory-managed string.
Definition: Ptexture.h:309
Ptex
Common data structures and enums used throughout the API.
Definition: Ptexture.h:71
PtexReader.h
PtexCache::Stats::fileReopens
uint64_t fileReopens
Definition: Ptexture.h:798
PtexHashMap::foreach
void foreach(Fn &fn)
Definition: PtexHashMap.h:252
PtexReaderCache::logRecentlyUsed
void logRecentlyUsed(PtexCachedReader *reader)
Definition: PtexCache.cpp:188
PtexReaderCache::_blockReads
size_t _blockReads
Definition: PtexCache.h:306
AtomicIncrement
PTEX_INLINE T AtomicIncrement(volatile T *target)
Definition: PtexPlatform.h:214
PtexTexture
Interface for reading data from a ptex file.
Definition: Ptexture.h:470
PtexReaderCache::_mruList
MruList *volatile _mruList
Definition: PtexCache.h:297
PtexCache.h
PtexReaderCache::Purger::memUsedChangeTotal
size_t memUsedChangeTotal
Definition: PtexCache.h:269
PtexReaderCache::_filesOpen
volatile size_t _filesOpen
Definition: PtexCache.h:288
PtexCache::Stats::filesOpen
uint64_t filesOpen
Definition: Ptexture.h:795
PtexReaderCache::_memUsed
volatile size_t _memUsed
Definition: PtexCache.h:287
PtexCachedReader
Definition: PtexCache.h:109
PtexReaderCache::adjustFilesOpen
void adjustFilesOpen(size_t amount)
Definition: PtexCache.h:259
PtexReaderCache::_err
PtexErrorHandler * _err
Definition: PtexCache.h:281
PtexReaderCache::Purger
Definition: PtexCache.h:268
PtexReaderCache::_peakMemUsed
size_t _peakMemUsed
Definition: PtexCache.h:303
PtexReader::invalidate
void invalidate()
Definition: PtexReader.h:66
PtexReader
Definition: PtexReader.h:52
PtexReader::logOpen
void logOpen()
Definition: PtexReader.h:72
PtexLruList::push
void push(T *node)
Definition: PtexCache.h:88
PtexReaderCache::_peakFilesOpen
size_t _peakFilesOpen
Definition: PtexCache.h:304
PtexCache::Stats
Definition: Ptexture.h:792
PtexReaderCache::findFile
bool findFile(const char *&filename, std::string &buffer, Ptex::String &error)
Definition: PtexCache.cpp:91
PtexReaderCache::_fileOpens
size_t _fileOpens
Definition: PtexCache.h:305
PtexReaderCache::processMru
void processMru()
Definition: PtexCache.cpp:203
Ptexture.h
Public API classes for reading, writing, caching, and filtering Ptex files.
set
PtexVersion h ONLY set(SRCS PtexCache.cpp PtexFilters.cpp PtexHalf.cpp PtexReader.cpp PtexSeparableFilter.cpp PtexSeparableKernel.cpp PtexTriangleFilter.cpp PtexTriangleKernel.cpp PtexUtils.cpp PtexWriter.cpp) if(PTEX_BUILD_STATIC_LIBS) add_library(Ptex_static STATIC $
Definition: ptex/CMakeLists.txt:8
Ptex::String::c_str
const char * c_str() const
Definition: Ptexture.h:317
StringKey
Definition: PtexHashMap.h:83
PtexCachedReader::tryPurge
bool tryPurge(size_t &memUsedChange)
Definition: PtexCache.h:162
set_target_properties
set_target_properties(Ptex_static PROPERTIES OUTPUT_NAME Ptex) target_include_directories(Ptex_static PUBLIC $< INSTALL_INTERFACE
Definition: ptex/CMakeLists.txt:22
PtexReaderCache::purge
virtual void purge(PtexTexture *)
Remove a texture file from the cache.
Definition: PtexCache.cpp:287
PtexReaderCache::MruList
Definition: PtexCache.h:292
PtexPlatform.h
Platform-specific classes, functions, and includes.
Mutex::unlock
void unlock()
Definition: PtexPlatform.h:132
PtexReaderCache::numMruFiles
static const int numMruFiles
Definition: PtexCache.h:291