1
votes

I realize the db is old and this is a long shot...

I've got an older project that has been running on Neo4j 2.1.6 with no issues. Yesterday I started getting [{Neo.DatabaseError.Statement.ExecutionFailure }] errors when I try to make updates to the graph using Cypher via the transaction endpoint.

I finally tracked down the problem to a couple of relationships attached to a couple of nodes that seem to be corrupted/invalid. Trying to delete them on the console just returns 'null' and trying to delete them via the transaction endpoint leads to:

{"code":"Neo.DatabaseError.Statement.ExecutionFailure","message":null,"stackTrace":"java.lang.NullPointerException
     org.neo4j.kernel.impl.nioneo.xa.RelationshipDeleter.updateNodesForDeletedRelationship(RelationshipDeleter.java:187)
     org.neo4j.kernel.impl.nioneo.xa.RelationshipDeleter.relDelete(RelationshipDeleter.java:67)
     org.neo4j.kernel.impl.nioneo.xa.NeoStoreTransactionContext.relationshipDelete(NeoStoreTransactionContext.java:85)
     org.neo4j.kernel.impl.nioneo.xa.NeoStoreTransaction.relDelete(NeoStoreTransaction.java:818)
     org.neo4j.kernel.impl.persistence.PersistenceManager.relDelete(PersistenceManager.java:140)
     org.neo4j.kernel.impl.core.NodeManager.deleteRelationship(NodeManager.java:780)
     org.neo4j.kernel.impl.api.state.OldTxStateBridgeImpl.deleteRelationship(OldTxStateBridgeImpl.java:164)
     org.neo4j.kernel.impl.api.state.TxStateImpl.relationshipDoDelete(TxStateImpl.java:439)
     org.neo4j.kernel.impl.api.StateHandlingStatementOperations$1.visit(StateHandlingStatementOperations.java:138)
     org.neo4j.kernel.impl.api.store.CacheLayer.visit(CacheLayer.java:476)
     org.neo4j.kernel.impl.api.StateHandlingStatementOperations.relationshipDelete(StateHandlingStatementOperations.java:133)
     org.neo4j.kernel.impl.api.ConstraintEnforcingEntityOperations.relationshipDelete(ConstraintEnforcingEntityOperations.java:159)
     org.neo4j.kernel.impl.api.LockingStatementOperations.relationshipDelete(LockingStatementOperations.java:231)
     org.neo4j.kernel.impl.api.OperationsFacade.relationshipDelete(OperationsFacade.java:528)
     org.neo4j.cypher.internal.spi.v2_1.TransactionBoundQueryContext$RelationshipOperations.delete(TransactionBoundQueryContext.scala:180)
     org.neo4j.cypher.internal.spi.v2_1.TransactionBoundQueryContext$RelationshipOperations.delete(TransactionBoundQueryContext.scala:178)
     org.neo4j.cypher.internal.compiler.v2_1.spi.DelegatingOperations.delete(DelegatingQueryContext.scala:110)
     org.neo4j.cypher.internal.compiler.v2_1.spi.ExceptionTranslatingQueryContext$ExceptionTranslatingOperations.org$neo4j$cypher$internal$compiler$v2_1$spi$ExceptionTranslatingQueryContext$ExceptionTranslatingOperations$$super$delete(ExceptionTranslatingQueryContext.scala:118)
     org.neo4j.cypher.internal.compiler.v2_1.spi.ExceptionTranslatingQueryContext$ExceptionTranslatingOperations$$anonfun$delete$1.apply$mcV$sp(ExceptionTranslatingQueryContext.scala:118)
     org.neo4j.cypher.internal.compiler.v2_1.spi.ExceptionTranslatingQueryContext$ExceptionTranslatingOperations$$anonfun$delete$1.apply(ExceptionTranslatingQueryContext.scala:118)
     org.neo4j.cypher.internal.compiler.v2_1.spi.ExceptionTranslatingQueryContext$ExceptionTranslatingOperations$$anonfun$delete$1.apply(ExceptionTranslatingQueryContext.scala:118)
     org.neo4j.cypher.internal.compiler.v2_1.spi.ExceptionTranslatingQueryContext.org$neo4j$cypher$internal$compiler$v2_1$spi$ExceptionTranslatingQueryContext$$translateException(ExceptionTranslatingQueryContext.scala:152)
     org.neo4j.cypher.internal.compiler.v2_1.spi.ExceptionTranslatingQueryContext$ExceptionTranslatingOperations.delete(ExceptionTranslatingQueryContext.scala:118)
     org.neo4j.cypher.internal.compiler.v2_1.spi.UpdateCountingQueryContext$CountingOps.delete(UpdateCountingQueryContext.scala:120)
     org.neo4j.cypher.internal.compiler.v2_1.mutation.DeleteEntityAction.org$neo4j$cypher$internal$compiler$v2_1$mutation$DeleteEntityAction$$delete(DeleteEntityAction.scala:51)
     org.neo4j.cypher.internal.compiler.v2_1.mutation.DeleteEntityAction.exec(DeleteEntityAction.scala:37)
     org.neo4j.cypher.internal.compiler.v2_1.pipes.ExecuteUpdateCommandsPipe.org$neo4j$cypher$internal$compiler$v2_1$pipes$ExecuteUpdateCommandsPipe$$exec(ExecuteUpdateCommandsPipe.scala:57)
     org.neo4j.cypher.internal.compiler.v2_1.pipes.ExecuteUpdateCommandsPi$$$$1019fdff8b266d7d9d5647386930b3d8$$$$ands$1$$anonfun$apply$2.apply(ExecuteUpdateCommandsPipe.scala:46)
     org.neo4j.cypher.internal.compiler.v2_1.pipes.ExecuteUpdateCommandsPi$$$$1019fdff8b266d7d9d5647386930b3d8$$$$ands$1$$anonfun$apply$2.apply(ExecuteUpdateCommandsPipe.scala:46)
     scala.collection.Iterator$$anon$13.hasNext(Iterator.scala:371)
     scala.collection.Iterator$$anon$13.hasNext(Iterator.scala:371)
     org.neo4j.cypher.internal.compiler.v2_1.pipes.EmptyResultPipe.internalCreateResults(EmptyResultPipe.scala:29)
     org.neo4j.cypher.internal.compiler.v2_1.pipes.PipeWithSource.createResults(Pipe.scala:105)
     org.neo4j.cypher.internal.compiler.v2_1.executionplan.ExecutionPlanBuilder$$anonfun$getExecutionPlanFunction$1$$anonfun$apply$2.apply(ExecutionPlanBuilder.scala:120)
     org.neo4j.cypher.internal.compiler.v2_1.executionplan.ExecutionPlanBuilder$$anonfun$getExecutionPlanFunction$1$$anonfun$apply$2.apply(ExecutionPlanBuilder.scala:119)
     org.neo4j.cypher.internal.compiler.v2_1.executionplan.ExecutionWorkflowBuilder.runWithQueryState(ExecutionPlanBuilder.scala:168)
     org.neo4j.cypher.internal.compiler.v2_1.executionplan.ExecutionPlanBuilder$$anonfun$getExecutionPlanFunction$1.apply(ExecutionPlanBuilder.scala:118)
     org.neo4j.cypher.internal.compiler.v2_1.executionplan.ExecutionPlanBuilder$$anonfun$getExecutionPlanFunction$1.apply(ExecutionPlanBuilder.scala:103)
     org.neo4j.cypher.internal.compiler.v2_1.executionplan.ExecutionPlanBuilder$$anon$1.execute(ExecutionPlanBuilder.scala:68)
     org.neo4j.cypher.internal.compiler.v2_1.executionplan.ExecutionPlanBuilder$$anon$1.execute(ExecutionPlanBuilder.scala:67)
     org.neo4j.cypher.internal.ExecutionPlanWrapperForV2_1.execute(CypherCompiler.scala:159)
     org.neo4j.cypher.ExecutionEngine.execute(ExecutionEngine.scala:76)
     org.neo4j.cypher.ExecutionEngine.execute(ExecutionEngine.scala:71)
     org.neo4j.cypher.javacompat.ExecutionEngine.execute(ExecutionEngine.java:84)
     org.neo4j.server.rest.transactional.TransactionHandle.executeStatements(TransactionHandle.java:277)
     org.neo4j.server.rest.transactional.TransactionHandle.commit(TransactionHandle.java:139)
     org.neo4j.server.rest.web.TransactionalService$2.write(TransactionalService.java:194)
     com.sun.jersey.core.impl.provider.entity.StreamingOutputProvider.writeTo(StreamingOutputProvider.java:71)
     com.sun.jersey.core.impl.provider.entity.StreamingOutputProvider.writeTo(StreamingOutputProvider.java:57)
     com.sun.jersey.spi.container.ContainerResponse.write(ContainerResponse.java:306)
     com.sun.jersey.server.impl.application.WebApplicationImpl._handleRequest(WebApplicationImpl.java:1437)
     com.sun.jersey.server.impl.application.WebApplicationImpl.handleRequest(WebApplicationImpl.java:1349)
     com.sun.jersey.server.impl.application.WebApplicationImpl.handleRequest(WebApplicationImpl.java:1339)
     com.sun.jersey.spi.container.servlet.WebComponent.service(WebComponent.java:416)
     com.sun.jersey.spi.container.servlet.ServletContainer.service(ServletContainer.java:537)
     com.sun.jersey.spi.container.servlet.ServletContainer.service(ServletContainer.java:699)
     javax.servlet.http.HttpServlet.service(HttpServlet.java:848)
     org.eclipse.jetty.servlet.ServletHolder.handle(ServletHolder.java:698)
     org.eclipse.jetty.servlet.ServletHandler$CachedChain.doFilter(ServletHandler.java:1506)
     org.neo4j.server.guard.GuardingRequestFilter.doFilter(GuardingRequestFilter.java:68)
     org.eclipse.jetty.servlet.ServletHandler$CachedChain.doFilter(ServletHandler.java:1477)
     ch.qos.logback.access.servlet.TeeFilter.doFilter(TeeFilter.java:55)
     org.eclipse.jetty.servlet.ServletHandler$CachedChain.doFilter(ServletHandler.java:1477)
     org.neo4j.server.guard.GuardingRequestFilter.doFilter(GuardingRequestFilter.java:68)
     org.eclipse.jetty.servlet.ServletHandler$CachedChain.doFilter(ServletHandler.java:1477)
     org.eclipse.jetty.servlet.ServletHandler.doHandle(ServletHandler.java:503)
     org.eclipse.jetty.server.session.SessionHandler.doHandle(SessionHandler.java:211)
     org.eclipse.jetty.server.handler.ContextHandler.doHandle(ContextHandler.java:1096)
     org.eclipse.jetty.servlet.ServletHandler.doScope(ServletHandler.java:432)
     org.eclipse.jetty.server.session.SessionHandler.doScope(SessionHandler.java:175)
     org.eclipse.jetty.server.handler.ContextHandler.doScope(ContextHandler.java:1030)
     org.eclipse.jetty.server.handler.ScopedHandler.handle(ScopedHandler.java:136)
     org.eclipse.jetty.server.handler.HandlerList.handle(HandlerList.java:52)
     org.eclipse.jetty.server.handler.HandlerWrapper.handle(HandlerWrapper.java:97)
     org.eclipse.jetty.server.handler.RequestLogHandler.handle(RequestLogHandler.java:92)
     org.eclipse.jetty.server.handler.HandlerWrapper.handle(HandlerWrapper.java:97)
     org.eclipse.jetty.server.Server.handle(Server.java:445)
     org.eclipse.jetty.server.HttpChannel.handle(HttpChannel.java:268)
     org.eclipse.jetty.server.HttpConnection.onFillable(HttpConnection.java:229)
     org.eclipse.jetty.io.AbstractConnection$ReadCallback.run(AbstractConnection.java:358)
     org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:601)
     org.eclipse.jetty.util.thread.QueuedThreadPool$3.run(QueuedThreadPool.java:532)
     java.lang.Thread.run(Thread.java:745)\n"

Is there anyway to nuke these relationships? Looks like the first one was corrupted two days ago and I'd prefer not to lose two days of data if at all possible.

1

1 Answers

1
votes

Just in case anybody else finds their way here, Michael Hunger wrote a tool that will copy a database and skip anything that is broken which is exactly what I needed. You can find it at https://github.com/jexp/store-utils/tree/21.

Turns out all of my issues were due to one relationship attached to a NULL node... Ran the tool, copied over the fixed up database and everything is working perfectly again.