platform/lang-impl/src/com/intellij/psi/stubs/StubUpdatingIndex.java - platform/tools/idea - Git at Google

 /*
  * Copyright 2000-2014 JetBrains s.r.o.
  *
  * Licensed under the Apache License, Version 2.0 (the "License");
  * you may not use this file except in compliance with the License.
  * You may obtain a copy of the License at
  *
  * http://www.apache.org/licenses/LICENSE-2.0
  *
  * Unless required by applicable law or agreed to in writing, software
  * distributed under the License is distributed on an "AS IS" BASIS,
  * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
  * See the License for the specific language governing permissions and
  * limitations under the License.
  */
 package com.intellij.psi.stubs;

 import com.intellij.lang.Language;
 import com.intellij.lang.LanguageParserDefinitions;
 import com.intellij.lang.ParserDefinition;
 import com.intellij.openapi.application.ApplicationManager;
 import com.intellij.openapi.diagnostic.Logger;
 import com.intellij.openapi.fileTypes.FileType;
 import com.intellij.openapi.fileTypes.FileTypeManager;
 import com.intellij.openapi.fileTypes.FileTypeRegistry;
 import com.intellij.openapi.fileTypes.LanguageFileType;
 import com.intellij.openapi.util.NotNullComputable;
 import com.intellij.openapi.util.io.BufferExposingByteArrayOutputStream;
 import com.intellij.openapi.vfs.VirtualFile;
 import com.intellij.openapi.vfs.newvfs.FileAttribute;
 import com.intellij.psi.tree.IFileElementType;
 import com.intellij.psi.tree.IStubFileElementType;
 import com.intellij.util.ExceptionUtil;
 import com.intellij.util.indexing.*;
 import com.intellij.util.io.DataExternalizer;
 import com.intellij.util.io.IntInlineKeyDescriptor;
 import com.intellij.util.io.KeyDescriptor;
 import org.jetbrains.annotations.NotNull;

 import java.io.*;
 import java.util.*;

 /*
  * @author max
  */
 public class StubUpdatingIndex extends CustomImplementationFileBasedIndexExtension<Integer, SerializedStubTree, FileContent> implements PsiDependentIndex {
   private static final Logger LOG = Logger.getInstance("#com.intellij.psi.stubs.StubUpdatingIndex");

   // todo remove once we don't need this for stub-ast mismatch debug info
   private static final FileAttribute INDEXED_STAMP = new FileAttribute("stubIndexStamp", 0, false);

   public static final ID<Integer, SerializedStubTree> INDEX_ID = ID.create("Stubs");

   private static final DataExternalizer<SerializedStubTree> KEY_EXTERNALIZER = new DataExternalizer<SerializedStubTree>() {
     @Override
     public void save(@NotNull final DataOutput out, @NotNull final SerializedStubTree v) throws IOException {
       v.write(out);
     }

     @NotNull
     @Override
     public SerializedStubTree read(@NotNull final DataInput in) throws IOException {
       return new SerializedStubTree(in);
     }
   };

   private static final FileBasedIndex.InputFilter INPUT_FILTER = new FileBasedIndex.InputFilter() {
     @Override
     public boolean acceptInput(@NotNull final VirtualFile file) {
       return canHaveStub(file);
     }
   };

   public static boolean canHaveStub(@NotNull VirtualFile file) {
     final FileType fileType = file.getFileType();
     if (fileType instanceof LanguageFileType) {
       final Language l = ((LanguageFileType)fileType).getLanguage();
       final ParserDefinition parserDefinition = LanguageParserDefinitions.INSTANCE.forLanguage(l);
       if (parserDefinition == null) {
         return false;
       }

       final IFileElementType elementType = parserDefinition.getFileNodeType();
       if (elementType instanceof IStubFileElementType) {
         if (((IStubFileElementType)elementType).shouldBuildStubFor(file)) {
           return true;
         }
         if (IndexingStamp.isFileIndexedStateCurrent(file, INDEX_ID)) {
           return true;
         }
       }
     }
     final BinaryFileStubBuilder builder = BinaryFileStubBuilders.INSTANCE.forFileType(fileType);
     return builder != null && builder.acceptsFile(file);
   }

   private static final KeyDescriptor<Integer> DATA_DESCRIPTOR = new IntInlineKeyDescriptor();

   @NotNull
   @Override
   public ID<Integer, SerializedStubTree> getName() {
     return INDEX_ID;
   }

   @Override
   public int getCacheSize() {
     return 5; // no need to cache many serialized trees
   }

   @Override
   public boolean keyIsUniqueForIndexedFile() {
     return true;
   }

   @NotNull
   @Override
   public DataIndexer<Integer, SerializedStubTree, FileContent> getIndexer() {
     return new DataIndexer<Integer, SerializedStubTree, FileContent>() {
       @Override
       @NotNull
       public Map<Integer, SerializedStubTree> map(@NotNull final FileContent inputData) {
         final Map<Integer, SerializedStubTree> result = new HashMap<Integer, SerializedStubTree>();

         ApplicationManager.getApplication().runReadAction(new Runnable() {
           @Override
           public void run() {
             final Stub rootStub = StubTreeBuilder.buildStubTree(inputData);
             if (rootStub == null) return;

             VirtualFile file = inputData.getFile();
             int contentLength = file.getFileType().isBinary() ? -1 : inputData.getContentAsText().length();
             rememberIndexingStamp(file, contentLength);

             final BufferExposingByteArrayOutputStream bytes = new BufferExposingByteArrayOutputStream();
             SerializationManagerEx.getInstanceEx().serialize(rootStub, bytes);

             final int key = Math.abs(FileBasedIndex.getFileId(file));
             result.put(key, new SerializedStubTree(bytes.getInternalBuffer(), bytes.size(), rootStub, file.getLength(), contentLength));
           }
         });

         return result;
       }
     };
   }

   private static void rememberIndexingStamp(final VirtualFile file, long contentLength) {
     try {
       DataOutputStream stream = INDEXED_STAMP.writeAttribute(file);
       stream.writeLong(file.getTimeStamp());
       stream.writeLong(contentLength);
       stream.close();
     }
     catch (IOException e) {
       LOG.error(e);
     }
   }

   public static String getIndexingStampInfo(VirtualFile file) {
     try {
       DataInputStream stream = INDEXED_STAMP.readAttribute(file);
       if (stream == null) {
         return "no data";
       }

       long stamp = stream.readLong();
       long size = stream.readLong();
       stream.close();
       return "indexed at " + stamp + " with size " + size;
     }
     catch (IOException e) {
       return ExceptionUtil.getThrowableText(e);
     }
   }

   @NotNull
   @Override
   public KeyDescriptor<Integer> getKeyDescriptor() {
     return DATA_DESCRIPTOR;
   }

   @NotNull
   @Override
   public DataExternalizer<SerializedStubTree> getValueExternalizer() {
     return KEY_EXTERNALIZER;
   }

   @NotNull
   @Override
   public FileBasedIndex.InputFilter getInputFilter() {
     return INPUT_FILTER;
   }

   @Override
   public boolean dependsOnFileContent() {
     return true;
   }

   @Override
   public int getVersion() {
     return CumulativeStubVersion.getCumulativeVersion();
   }

   @NotNull
   @Override
   public UpdatableIndex<Integer, SerializedStubTree, FileContent> createIndexImplementation(@NotNull final ID<Integer, SerializedStubTree> indexId, @NotNull final FileBasedIndex owner, @NotNull IndexStorage<Integer, SerializedStubTree> storage)
     throws StorageException, IOException {
     if (storage instanceof MemoryIndexStorage) {
       final MemoryIndexStorage<Integer, SerializedStubTree> memStorage = (MemoryIndexStorage<Integer, SerializedStubTree>)storage;
       memStorage.addBufferingStateListener(new MemoryIndexStorage.BufferingStateListener() {
         @Override
         public void bufferingStateChanged(final boolean newState) {
           ((StubIndexImpl)StubIndex.getInstance()).setDataBufferingEnabled(newState);
         }

         @Override
         public void memoryStorageCleared() {
           ((StubIndexImpl)StubIndex.getInstance()).cleanupMemoryStorage();
         }
       });
     }
     return new MyIndex(indexId, storage, getIndexer());
   }

   private static void updateStubIndices(@NotNull final Collection<StubIndexKey> indexKeys,
                                         final int inputId,
                                         @NotNull final Map<StubIndexKey, Map<Object, StubIdList>> oldStubTree,
                                         @NotNull final Map<StubIndexKey, Map<Object, StubIdList>> newStubTree) {
     final StubIndexImpl stubIndex = (StubIndexImpl)StubIndex.getInstance();
     for (StubIndexKey key : indexKeys) {
       final Map<Object, StubIdList> oldMap = oldStubTree.get(key);
       final Map<Object, StubIdList> newMap = newStubTree.get(key);

       final Map<Object, StubIdList> _oldMap = oldMap != null ? oldMap : Collections.<Object, StubIdList>emptyMap();
       final Map<Object, StubIdList> _newMap = newMap != null ? newMap : Collections.<Object, StubIdList>emptyMap();

       stubIndex.updateIndex(key, inputId, _oldMap, _newMap);
     }
   }

   @NotNull
   private static Collection<StubIndexKey> getAffectedIndices(@NotNull final Map<StubIndexKey, Map<Object, StubIdList>> oldStubTree,
                                                              @NotNull final Map<StubIndexKey, Map<Object, StubIdList>> newStubTree) {
     Set<StubIndexKey> allIndices = new HashSet<StubIndexKey>();
     allIndices.addAll(oldStubTree.keySet());
     allIndices.addAll(newStubTree.keySet());
     return allIndices;
   }

   private class MyIndex extends MapReduceIndex<Integer, SerializedStubTree, FileContent> {
     private StubIndexImpl myStubIndex;

     public MyIndex(final ID<Integer, SerializedStubTree> indexId, final IndexStorage<Integer, SerializedStubTree> storage, final DataIndexer<Integer, SerializedStubTree, FileContent> indexer)
       throws StorageException, IOException {
       super(indexId, indexer, storage);
       checkNameStorage();
     }

     @Override
     public void flush() throws StorageException {
       final StubIndexImpl stubIndex = getStubIndex();
       try {
         for (StubIndexKey key : stubIndex.getAllStubIndexKeys()) {
           stubIndex.flush(key);
         }
       }
       finally {
         super.flush();
       }
     }

     @Override
     protected void updateWithMap(final int inputId,
                                  int savedInputId, @NotNull final Map<Integer, SerializedStubTree> newData,
                                  @NotNull NotNullComputable<Collection<Integer>> oldKeysGetter)
       throws StorageException {

       checkNameStorage();
       final Map<StubIndexKey, Map<Object, StubIdList>> newStubTree;
       try {
         newStubTree = getStubTree(newData);
       }
       catch (SerializerNotFoundException e) {
         throw new StorageException(e);
       }

       final StubIndexImpl stubIndex = getStubIndex();
       final Collection<StubIndexKey> allStubIndices = stubIndex.getAllStubIndexKeys();
       try {
         // first write-lock affected stub indices to avoid deadlocks
         for (StubIndexKey key : allStubIndices) {
           stubIndex.getWriteLock(key).lock();
         }

         try {
           getWriteLock().lock();

           final Map<Integer, SerializedStubTree> oldData = readOldData(inputId);

           final Map<StubIndexKey, Map<Object, StubIdList>> oldStubTree;
           try {
             oldStubTree = getStubTree(oldData);
           }
           catch (SerializerNotFoundException e) {
             throw new StorageException(e);
           }

           super.updateWithMap(inputId, savedInputId, newData, oldKeysGetter);

           updateStubIndices(getAffectedIndices(oldStubTree, newStubTree), inputId, oldStubTree, newStubTree);
         }
         finally {
           getWriteLock().unlock();
         }
       }
       finally {
         for (StubIndexKey key : allStubIndices) {
           stubIndex.getWriteLock(key).unlock();
         }
       }
     }

     private StubIndexImpl getStubIndex() {
       StubIndexImpl index = myStubIndex;
       if (index == null) {
         index = myStubIndex = (StubIndexImpl)StubIndex.getInstance();
       }
       return index;
     }

     private void checkNameStorage() throws StorageException {
       final SerializationManagerEx serializationManager = SerializationManagerEx.getInstanceEx();
       if (serializationManager.isNameStorageCorrupted()) {
         serializationManager.repairNameStorage();
         //noinspection ThrowFromFinallyBlock
         throw new StorageException("NameStorage for stubs serialization has been corrupted");
       }
     }

     private Map<StubIndexKey, Map<Object, StubIdList>> getStubTree(@NotNull final Map<Integer, SerializedStubTree> data)
       throws SerializerNotFoundException {
       final Map<StubIndexKey, Map<Object, StubIdList>> stubTree;
       if (!data.isEmpty()) {
         final SerializedStubTree stub = data.values().iterator().next();
         ObjectStubBase root = (ObjectStubBase)stub.getStub(true);
         Map<StubIndexKey, Map<Object, int[]>> map = new ObjectStubTree(root, false).indexStubTree();

         // xxx:fix refs inplace
         stubTree = (Map)map;
         for(StubIndexKey key:map.keySet()) {
           Map<Object, int[]> value = map.get(key);
           for(Object k: value.keySet()) {
             int[] ints = value.get(k);
             StubIdList stubList = ints.length == 1 ? new StubIdList(ints[0]) : new StubIdList(ints, ints.length);
             ((Map<Object, StubIdList>)(Map)value).put(k, stubList);
           }
         }
       }
       else {
         stubTree = Collections.emptyMap();
       }
       return stubTree;
     }

     /*MUST be called under the WriteLock*/
     @NotNull
     private Map<Integer, SerializedStubTree> readOldData(final int key) throws StorageException {
       final Map<Integer, SerializedStubTree> result = new HashMap<Integer, SerializedStubTree>();

       IndexStorage<Integer, SerializedStubTree> indexStorage = myStorage;
       if (indexStorage instanceof MemoryIndexStorage) {
         final MemoryIndexStorage<Integer, SerializedStubTree> memIndexStorage = (MemoryIndexStorage<Integer, SerializedStubTree>)indexStorage;
         if (!memIndexStorage.isBufferingEnabled()) {
           // if buffering is not enabled, use backend storage to make sure
           // the returned stub tree contains no data corresponding to unsaved documents.
           // This will ensure that correct set of old keys is used for update
           indexStorage = memIndexStorage.getBackendStorage();
         }
       }
       try {
         final ValueContainer<SerializedStubTree> valueContainer = indexStorage.read(key);
         if (valueContainer.size() != 1) {
           LOG.assertTrue(valueContainer.size() == 0);
           return result;
         }

         result.put(key, valueContainer.getValueIterator().next());
         return result;
       }
       catch (RuntimeException e) {
         final Throwable cause = e.getCause();
         if (cause instanceof IOException) {
           throw new StorageException(cause);
         }
         throw e;
       }
     }

     @Override
     public void clear() throws StorageException {
       final StubIndexImpl stubIndex = StubIndexImpl.getInstanceOrInvalidate();
       final Collection<StubIndexKey> allStubIndexKeys = stubIndex != null? stubIndex.getAllStubIndexKeys() : Collections.<StubIndexKey>emptyList();
       try {
         for (StubIndexKey key : allStubIndexKeys) {
           //noinspection ConstantConditions
           stubIndex.getWriteLock(key).lock();
         }
         getWriteLock().lock();
         if (stubIndex != null) {
           stubIndex.clearAllIndices();
         }
         super.clear();
       }
       finally {
         getWriteLock().unlock();
         for (StubIndexKey key : allStubIndexKeys) {
           //noinspection ConstantConditions
           stubIndex.getWriteLock(key).unlock();
         }
       }
     }

     @Override
     public void dispose() {
       try {
         super.dispose();
       }
       finally {
         getStubIndex().dispose();
       }
     }
   }
 }
	/*
	* Copyright 2000-2014 JetBrains s.r.o.
	*
	* Licensed under the Apache License, Version 2.0 (the "License");
	* you may not use this file except in compliance with the License.
	* You may obtain a copy of the License at
	*
	* http://www.apache.org/licenses/LICENSE-2.0
	*
	* Unless required by applicable law or agreed to in writing, software
	* distributed under the License is distributed on an "AS IS" BASIS,
	* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
	* See the License for the specific language governing permissions and
	* limitations under the License.
	*/
	package com.intellij.psi.stubs;

	import com.intellij.lang.Language;
	import com.intellij.lang.LanguageParserDefinitions;
	import com.intellij.lang.ParserDefinition;
	import com.intellij.openapi.application.ApplicationManager;
	import com.intellij.openapi.diagnostic.Logger;
	import com.intellij.openapi.fileTypes.FileType;
	import com.intellij.openapi.fileTypes.FileTypeManager;
	import com.intellij.openapi.fileTypes.FileTypeRegistry;
	import com.intellij.openapi.fileTypes.LanguageFileType;
	import com.intellij.openapi.util.NotNullComputable;
	import com.intellij.openapi.util.io.BufferExposingByteArrayOutputStream;
	import com.intellij.openapi.vfs.VirtualFile;
	import com.intellij.openapi.vfs.newvfs.FileAttribute;
	import com.intellij.psi.tree.IFileElementType;
	import com.intellij.psi.tree.IStubFileElementType;
	import com.intellij.util.ExceptionUtil;
	import com.intellij.util.indexing.*;
	import com.intellij.util.io.DataExternalizer;
	import com.intellij.util.io.IntInlineKeyDescriptor;
	import com.intellij.util.io.KeyDescriptor;
	import org.jetbrains.annotations.NotNull;

	import java.io.*;
	import java.util.*;

	/*
	* @author max
	*/
	public class StubUpdatingIndex extends CustomImplementationFileBasedIndexExtension<Integer, SerializedStubTree, FileContent> implements PsiDependentIndex {
	private static final Logger LOG = Logger.getInstance("#com.intellij.psi.stubs.StubUpdatingIndex");

	// todo remove once we don't need this for stub-ast mismatch debug info
	private static final FileAttribute INDEXED_STAMP = new FileAttribute("stubIndexStamp", 0, false);

	public static final ID<Integer, SerializedStubTree> INDEX_ID = ID.create("Stubs");

	private static final DataExternalizer<SerializedStubTree> KEY_EXTERNALIZER = new DataExternalizer<SerializedStubTree>() {
	@Override
	public void save(@NotNull final DataOutput out, @NotNull final SerializedStubTree v) throws IOException {
	v.write(out);
	}

	@NotNull
	@Override
	public SerializedStubTree read(@NotNull final DataInput in) throws IOException {
	return new SerializedStubTree(in);
	}
	};

	private static final FileBasedIndex.InputFilter INPUT_FILTER = new FileBasedIndex.InputFilter() {
	@Override
	public boolean acceptInput(@NotNull final VirtualFile file) {
	return canHaveStub(file);
	}
	};

	public static boolean canHaveStub(@NotNull VirtualFile file) {
	final FileType fileType = file.getFileType();
	if (fileType instanceof LanguageFileType) {
	final Language l = ((LanguageFileType)fileType).getLanguage();
	final ParserDefinition parserDefinition = LanguageParserDefinitions.INSTANCE.forLanguage(l);
	if (parserDefinition == null) {
	return false;
	}

	final IFileElementType elementType = parserDefinition.getFileNodeType();
	if (elementType instanceof IStubFileElementType) {
	if (((IStubFileElementType)elementType).shouldBuildStubFor(file)) {
	return true;
	}
	if (IndexingStamp.isFileIndexedStateCurrent(file, INDEX_ID)) {
	return true;
	}
	}
	}
	final BinaryFileStubBuilder builder = BinaryFileStubBuilders.INSTANCE.forFileType(fileType);
	return builder != null && builder.acceptsFile(file);
	}

	private static final KeyDescriptor<Integer> DATA_DESCRIPTOR = new IntInlineKeyDescriptor();

	@NotNull
	@Override
	public ID<Integer, SerializedStubTree> getName() {
	return INDEX_ID;
	}

	@Override
	public int getCacheSize() {
	return 5; // no need to cache many serialized trees
	}

	@Override
	public boolean keyIsUniqueForIndexedFile() {
	return true;
	}

	@NotNull
	@Override
	public DataIndexer<Integer, SerializedStubTree, FileContent> getIndexer() {
	return new DataIndexer<Integer, SerializedStubTree, FileContent>() {
	@Override
	@NotNull
	public Map<Integer, SerializedStubTree> map(@NotNull final FileContent inputData) {
	final Map<Integer, SerializedStubTree> result = new HashMap<Integer, SerializedStubTree>();

	ApplicationManager.getApplication().runReadAction(new Runnable() {
	@Override
	public void run() {
	final Stub rootStub = StubTreeBuilder.buildStubTree(inputData);
	if (rootStub == null) return;

	VirtualFile file = inputData.getFile();
	int contentLength = file.getFileType().isBinary() ? -1 : inputData.getContentAsText().length();
	rememberIndexingStamp(file, contentLength);

	final BufferExposingByteArrayOutputStream bytes = new BufferExposingByteArrayOutputStream();
	SerializationManagerEx.getInstanceEx().serialize(rootStub, bytes);

	final int key = Math.abs(FileBasedIndex.getFileId(file));
	result.put(key, new SerializedStubTree(bytes.getInternalBuffer(), bytes.size(), rootStub, file.getLength(), contentLength));
	}
	});

	return result;
	}
	};
	}

	private static void rememberIndexingStamp(final VirtualFile file, long contentLength) {
	try {
	DataOutputStream stream = INDEXED_STAMP.writeAttribute(file);
	stream.writeLong(file.getTimeStamp());
	stream.writeLong(contentLength);
	stream.close();
	}
	catch (IOException e) {
	LOG.error(e);
	}
	}

	public static String getIndexingStampInfo(VirtualFile file) {
	try {
	DataInputStream stream = INDEXED_STAMP.readAttribute(file);
	if (stream == null) {
	return "no data";
	}

	long stamp = stream.readLong();
	long size = stream.readLong();
	stream.close();
	return "indexed at " + stamp + " with size " + size;
	}
	catch (IOException e) {
	return ExceptionUtil.getThrowableText(e);
	}
	}

	@NotNull
	@Override
	public KeyDescriptor<Integer> getKeyDescriptor() {
	return DATA_DESCRIPTOR;
	}

	@NotNull
	@Override
	public DataExternalizer<SerializedStubTree> getValueExternalizer() {
	return KEY_EXTERNALIZER;
	}

	@NotNull
	@Override
	public FileBasedIndex.InputFilter getInputFilter() {
	return INPUT_FILTER;
	}

	@Override
	public boolean dependsOnFileContent() {
	return true;
	}

	@Override
	public int getVersion() {
	return CumulativeStubVersion.getCumulativeVersion();
	}

	@NotNull
	@Override
	public UpdatableIndex<Integer, SerializedStubTree, FileContent> createIndexImplementation(@NotNull final ID<Integer, SerializedStubTree> indexId, @NotNull final FileBasedIndex owner, @NotNull IndexStorage<Integer, SerializedStubTree> storage)
	throws StorageException, IOException {
	if (storage instanceof MemoryIndexStorage) {
	final MemoryIndexStorage<Integer, SerializedStubTree> memStorage = (MemoryIndexStorage<Integer, SerializedStubTree>)storage;
	memStorage.addBufferingStateListener(new MemoryIndexStorage.BufferingStateListener() {
	@Override
	public void bufferingStateChanged(final boolean newState) {
	((StubIndexImpl)StubIndex.getInstance()).setDataBufferingEnabled(newState);
	}

	@Override
	public void memoryStorageCleared() {
	((StubIndexImpl)StubIndex.getInstance()).cleanupMemoryStorage();
	}
	});
	}
	return new MyIndex(indexId, storage, getIndexer());
	}

	private static void updateStubIndices(@NotNull final Collection<StubIndexKey> indexKeys,
	final int inputId,
	@NotNull final Map<StubIndexKey, Map<Object, StubIdList>> oldStubTree,
	@NotNull final Map<StubIndexKey, Map<Object, StubIdList>> newStubTree) {
	final StubIndexImpl stubIndex = (StubIndexImpl)StubIndex.getInstance();
	for (StubIndexKey key : indexKeys) {
	final Map<Object, StubIdList> oldMap = oldStubTree.get(key);
	final Map<Object, StubIdList> newMap = newStubTree.get(key);

	final Map<Object, StubIdList> _oldMap = oldMap != null ? oldMap : Collections.<Object, StubIdList>emptyMap();
	final Map<Object, StubIdList> _newMap = newMap != null ? newMap : Collections.<Object, StubIdList>emptyMap();

	stubIndex.updateIndex(key, inputId, _oldMap, _newMap);
	}
	}

	@NotNull
	private static Collection<StubIndexKey> getAffectedIndices(@NotNull final Map<StubIndexKey, Map<Object, StubIdList>> oldStubTree,
	@NotNull final Map<StubIndexKey, Map<Object, StubIdList>> newStubTree) {
	Set<StubIndexKey> allIndices = new HashSet<StubIndexKey>();
	allIndices.addAll(oldStubTree.keySet());
	allIndices.addAll(newStubTree.keySet());
	return allIndices;
	}

	private class MyIndex extends MapReduceIndex<Integer, SerializedStubTree, FileContent> {
	private StubIndexImpl myStubIndex;

	public MyIndex(final ID<Integer, SerializedStubTree> indexId, final IndexStorage<Integer, SerializedStubTree> storage, final DataIndexer<Integer, SerializedStubTree, FileContent> indexer)
	throws StorageException, IOException {
	super(indexId, indexer, storage);
	checkNameStorage();
	}

	@Override
	public void flush() throws StorageException {
	final StubIndexImpl stubIndex = getStubIndex();
	try {
	for (StubIndexKey key : stubIndex.getAllStubIndexKeys()) {
	stubIndex.flush(key);
	}
	}
	finally {
	super.flush();
	}
	}

	@Override
	protected void updateWithMap(final int inputId,
	int savedInputId, @NotNull final Map<Integer, SerializedStubTree> newData,
	@NotNull NotNullComputable<Collection<Integer>> oldKeysGetter)
	throws StorageException {

	checkNameStorage();
	final Map<StubIndexKey, Map<Object, StubIdList>> newStubTree;
	try {
	newStubTree = getStubTree(newData);
	}
	catch (SerializerNotFoundException e) {
	throw new StorageException(e);
	}

	final StubIndexImpl stubIndex = getStubIndex();
	final Collection<StubIndexKey> allStubIndices = stubIndex.getAllStubIndexKeys();
	try {
	// first write-lock affected stub indices to avoid deadlocks
	for (StubIndexKey key : allStubIndices) {
	stubIndex.getWriteLock(key).lock();
	}

	try {
	getWriteLock().lock();

	final Map<Integer, SerializedStubTree> oldData = readOldData(inputId);

	final Map<StubIndexKey, Map<Object, StubIdList>> oldStubTree;
	try {
	oldStubTree = getStubTree(oldData);
	}
	catch (SerializerNotFoundException e) {
	throw new StorageException(e);
	}

	super.updateWithMap(inputId, savedInputId, newData, oldKeysGetter);

	updateStubIndices(getAffectedIndices(oldStubTree, newStubTree), inputId, oldStubTree, newStubTree);
	}
	finally {
	getWriteLock().unlock();
	}
	}
	finally {
	for (StubIndexKey key : allStubIndices) {
	stubIndex.getWriteLock(key).unlock();
	}
	}
	}

	private StubIndexImpl getStubIndex() {
	StubIndexImpl index = myStubIndex;
	if (index == null) {
	index = myStubIndex = (StubIndexImpl)StubIndex.getInstance();
	}
	return index;
	}

	private void checkNameStorage() throws StorageException {
	final SerializationManagerEx serializationManager = SerializationManagerEx.getInstanceEx();
	if (serializationManager.isNameStorageCorrupted()) {
	serializationManager.repairNameStorage();
	//noinspection ThrowFromFinallyBlock
	throw new StorageException("NameStorage for stubs serialization has been corrupted");
	}
	}

	private Map<StubIndexKey, Map<Object, StubIdList>> getStubTree(@NotNull final Map<Integer, SerializedStubTree> data)
	throws SerializerNotFoundException {
	final Map<StubIndexKey, Map<Object, StubIdList>> stubTree;
	if (!data.isEmpty()) {
	final SerializedStubTree stub = data.values().iterator().next();
	ObjectStubBase root = (ObjectStubBase)stub.getStub(true);
	Map<StubIndexKey, Map<Object, int[]>> map = new ObjectStubTree(root, false).indexStubTree();

	// xxx:fix refs inplace
	stubTree = (Map)map;
	for(StubIndexKey key:map.keySet()) {
	Map<Object, int[]> value = map.get(key);
	for(Object k: value.keySet()) {
	int[] ints = value.get(k);
	StubIdList stubList = ints.length == 1 ? new StubIdList(ints[0]) : new StubIdList(ints, ints.length);
	((Map<Object, StubIdList>)(Map)value).put(k, stubList);
	}
	}
	}
	else {
	stubTree = Collections.emptyMap();
	}
	return stubTree;
	}

	/MUST be called under the WriteLock/
	@NotNull
	private Map<Integer, SerializedStubTree> readOldData(final int key) throws StorageException {
	final Map<Integer, SerializedStubTree> result = new HashMap<Integer, SerializedStubTree>();

	IndexStorage<Integer, SerializedStubTree> indexStorage = myStorage;
	if (indexStorage instanceof MemoryIndexStorage) {
	final MemoryIndexStorage<Integer, SerializedStubTree> memIndexStorage = (MemoryIndexStorage<Integer, SerializedStubTree>)indexStorage;
	if (!memIndexStorage.isBufferingEnabled()) {
	// if buffering is not enabled, use backend storage to make sure
	// the returned stub tree contains no data corresponding to unsaved documents.
	// This will ensure that correct set of old keys is used for update
	indexStorage = memIndexStorage.getBackendStorage();
	}
	}
	try {
	final ValueContainer<SerializedStubTree> valueContainer = indexStorage.read(key);
	if (valueContainer.size() != 1) {
	LOG.assertTrue(valueContainer.size() == 0);
	return result;
	}

	result.put(key, valueContainer.getValueIterator().next());
	return result;
	}
	catch (RuntimeException e) {
	final Throwable cause = e.getCause();
	if (cause instanceof IOException) {
	throw new StorageException(cause);
	}
	throw e;
	}
	}

	@Override
	public void clear() throws StorageException {
	final StubIndexImpl stubIndex = StubIndexImpl.getInstanceOrInvalidate();
	final Collection<StubIndexKey> allStubIndexKeys = stubIndex != null? stubIndex.getAllStubIndexKeys() : Collections.<StubIndexKey>emptyList();
	try {
	for (StubIndexKey key : allStubIndexKeys) {
	//noinspection ConstantConditions
	stubIndex.getWriteLock(key).lock();
	}
	getWriteLock().lock();
	if (stubIndex != null) {
	stubIndex.clearAllIndices();
	}
	super.clear();
	}
	finally {
	getWriteLock().unlock();
	for (StubIndexKey key : allStubIndexKeys) {
	//noinspection ConstantConditions
	stubIndex.getWriteLock(key).unlock();
	}
	}
	}

	@Override
	public void dispose() {
	try {
	super.dispose();
	}
	finally {
	getStubIndex().dispose();
	}
	}
	}
	}