Table of Contents

Pixel raycast

Suppose you want to know what block or entity corresponds to a pixel on the screen. This can be done with pixel raycast.

All of this is client side.

There are two cases, center pixel (crosshair) and arbitrary pixel.

Special case: Center pixel

This can be done with: 

  1. MinecraftClient client = MinecraftClient.getInstance();
  2. HitResult hit = client.crosshairTarget;
  3.  
  4. switch(hit.getType()) {
  5.     case Type.MISS:
  6.         //nothing near enough
  7.         break; 
  8.     case Type.BLOCK:
  9.         BlockHitResult blockHit = (BlockHitResult) hit;
  10.         BlockPos blockPos = blockHit.getBlockPos();
  11.         BlockState blockState = client.world.getBlockState(blockPos);
  12. 	Block block = blockState.getBlock();
  13.         break; 
  14.     case Type.ENTITY:
  15.         EntityHitResult entityHit = (EntityHitResult) hit;
  16.         Entity entity = entityHit.getEntity();
  17.         break; 
  18. }

For arbitrary reach

The code above allows for the normal reach, 3 blocks for survival and 4.5 in creative. If you want the raycast to reach farther you need to replace 

 HitResult hit = client.crosshairTarget;

in Special Case: Center Pixel with the example below: 

  1. double maxReach = 1000; //The farthest target the cameraEntity can detect
  2. float tickDelta = 1.0F; //Used for tracking animation progress; no tracking is 1.0F
  3. boolean includeFluids = true; //Whether to detect fluids as blocks
  4.  
  5. HitResult hit = client.cameraEntity.raycast(maxReach, tickDelta, includeFluids);

General case: Arbitrary pixel

Example here.

For this one we need to precalculate a few things: 

  1. MinecraftClient client = MinecraftClient.getInstance();
  2. int width = client.getWindow().getScaledWidth();
  3. int height = client.getWindow().getScaledHeight();
  4. Vec3d cameraDirection = client.cameraEntity.getRotationVec(tickDelta);
  5. double fov = client.options.fov;
  6. double angleSize = fov/height;
  7. Vector3f verticalRotationAxis = new Vector3f(cameraDirection);
  8. verticalRotationAxis.cross(Vector3f.POSITIVE_Y);
  9. if(!verticalRotationAxis.normalize()) {
  10.     return;//The camera is pointing directly up or down, you'll have to fix this one
  11. }
  12.  
  13. Vector3f horizontalRotationAxis = new Vector3f(cameraDirection);
  14. horizontalRotationAxis.cross(verticalRotationAxis);
  15. horizontalRotationAxis.normalize();
  16.  
  17. verticalRotationAxis = new Vector3f(cameraDirection);
  18. verticalRotationAxis.cross(horizontalRotationAxis);

Once you have that, you can use this function to calculate the direction vector that corresponds to the pixel: 

  1. private static Vec3d map(float anglePerPixel, Vec3d center, Vector3f horizontalRotationAxis,
  2.     Vector3f verticalRotationAxis, int x, int y, int width, int height) {
  3.     float horizontalRotation = (x - width/2f) * anglePerPixel;
  4.     float verticalRotation = (y - height/2f) * anglePerPixel;
  5.  
  6.     final Vector3f temp2 = new Vector3f(center);
  7.     temp2.rotate(verticalRotationAxis.getDegreesQuaternion(verticalRotation));
  8.     temp2.rotate(horizontalRotationAxis.getDegreesQuaternion(horizontalRotation));
  9.     return new Vec3d(temp2);
  10. }

Then you have this reimplementation of the code at GameRenderer#updateTargetedEntity: 

  1. private static HitResult raycastInDirection(MinecraftClient client, float tickDelta, Vec3d direction) {
  2.     Entity entity = client.getCameraEntity();
  3.     if (entity == null || client.world == null) {
  4.         return null;
  5.     }
  6.  
  7.     double reachDistance = client.interactionManager.getReachDistance();//Change this to extend the reach
  8.     HitResult target = raycast(entity, reachDistance, tickDelta, false, direction);
  9.     boolean tooFar = false;
  10.     double extendedReach = reachDistance;
  11.     if (client.interactionManager.hasExtendedReach()) {
  12.         extendedReach = 6.0D;//Change this to extend the reach
  13.         reachDistance = extendedReach;
  14.     } else {
  15.         if (reachDistance > 3.0D) {
  16.             tooFar = true;
  17.         }
  18.     }
  19.  
  20.     Vec3d cameraPos = entity.getCameraPosVec(tickDelta);
  21.  
  22.     extendedReach = extendedReach * extendedReach;
  23.     if (target != null) {
  24.         extendedReach = target.getPos().squaredDistanceTo(cameraPos);
  25.     }
  26.  
  27.     Vec3d vec3d3 = cameraPos.add(direction.multiply(reachDistance));
  28.     Box box = entity
  29.             .getBoundingBox()
  30.             .stretch(entity.getRotationVec(1.0F).multiply(reachDistance))
  31.             .expand(1.0D, 1.0D, 1.0D);
  32.     EntityHitResult entityHitResult = ProjectileUtil.raycast(
  33.             entity,
  34.             cameraPos,
  35.             vec3d3,
  36.             box,
  37.             (entityx) -> !entityx.isSpectator() && entityx.collides(),
  38.             extendedReach
  39.     );
  40.  
  41.     if (entityHitResult == null) {
  42.         return target;
  43.     }
  44.  
  45.     Entity entity2 = entityHitResult.getEntity();
  46.     Vec3d vec3d4 = entityHitResult.getPos();
  47.     double g = cameraPos.squaredDistanceTo(vec3d4);
  48.     if (tooFar && g > 9.0D) {
  49.         return null;
  50.     } else if (g < extendedReach || target == null) {
  51.         target = entityHitResult;
  52.         if (entity2 instanceof LivingEntity || entity2 instanceof ItemFrameEntity) {
  53.             client.targetedEntity = entity2;
  54.         }
  55.     }
  56.  
  57.     return target;
  58. }
  59.  
  60. private static HitResult raycast(
  61.         Entity entity,
  62.         double maxDistance,
  63.         float tickDelta,
  64.         boolean includeFluids,
  65.         Vec3d direction
  66. ) {
  67.     Vec3d end = entity.getCameraPosVec(tickDelta).add(direction.multiply(maxDistance));
  68.     return entity.world.raycast(new RaycastContext(
  69.             entity.getCameraPosVec(tickDelta),
  70.             end,
  71.             RaycastContext.ShapeType.OUTLINE,
  72.             includeFluids ? RaycastContext.FluidHandling.ANY : RaycastContext.FluidHandling.NONE,
  73.             entity
  74.     ));
  75. }

Once you have the direction and the raycaster, you can put them together: 

  1. Vec3d direction = map(
  2.         (float) angleSize,
  3.         cameraDirection,
  4.         horizontalRotationAxis,
  5.         verticalRotationAxis,
  6.         x,
  7.         y,
  8.         width,
  9.         height
  10. );
  11. HitResult hit = raycastInDirection(client, tickDelta, direction);
  12.  
  13. switch(hit.getType()) {
  14.     case Type.MISS:
  15.         //nothing near enough
  16.         break; 
  17.     case Type.BLOCK:
  18.         BlockHitResult blockHit = (BlockHitResult) hit;
  19.         BlockPos blockPos = blockHit.getBlockPos();
  20.         BlockState blockState = client.world.getBlockState(blockPos);
  21. 	Block block = blockState.getBlock();
  22.         break; 
  23.     case Type.ENTITY:
  24.         EntityHitResult entityHit = (EntityHitResult) hit;
  25.         Entity entity = entityHit.getEntity();
  26.         break; 
  27. }

Here x and y are your pixel coordinates.

Performance considerations

This is EXPENSIVE, if you do it too many times, it WILL get slow. Especially for long reaches. If you need to do many raycasts in a single frame, this link might be helpful.