Raycasts stop working after ~3 seconds

So in the project i'm working on you're suppose to be able to look at an object and pick it up, I've got that pretty much figured out but when I play the game i can pick up any object I like but after a little bit of time the object will fall out of my "hands". As per this I thought it had to do with rigidbodies so I removed them to no avail. As as per this I thought it had to do with Capsule Colliders, but a lass Unity's standard FPS controller does not use them.
The symptoms of my problem can be seen here. In the video the both boxes have a ridgidbody component but the second is is kinamatic. It seems like the raycast does not register that it should be hitting anything for an instant, letting the box go, but im not sure. Any ideas?

Also heres the relevant code:

 using System.Collections;
 using System.Collections.Generic;
 using UnityEngine;
 using UnityEngine.UI;
 using UnityEngine.EventSystems;
 
 public class CrossHair : MonoBehaviour {
     #region variables
     [Header("Colors of crosshair")]
     [Tooltip("Colour when you're not touching something, RGBA")]
     public Vector4 default_colour;
     [Tooltip("Colour when you're touching something, RGBA")]
     public Vector4 highlight_colour;
 
     private Image cross_hair_img_component;
 
     //stuff for the raycast
     public Camera raycast_camera;
     public float crosshair_range;
     RaycastHit crosshair_hit;
 
     //stuff relating to the object being picked up
     [Header("Crosshair hit's object's properties")]
     [Tooltip("distance between the object and the camera when picking the object up")]
     [SerializeField] private float object_distance;  //object distance
     private bool is_Interactive;    //true if object is interactive
     #endregion
 
     // Use this for initialization
     void Start () {
         #region gets cross_hair_img_component
         //gets the gets the child (cross_hair_img) of canvas_for_cross_hair in the hiarchary
         //which is a child of the gameobject (FirstPersonCharacter) then gets the Image component of that child
         Transform canvas_in_hierarchy = gameObject.transform.Find("canvas_for_cross_hair");
         Transform cross_hair_in_hierarchy = canvas_in_hierarchy.transform.Find("cross_hair_img");
         cross_hair_img_component = cross_hair_in_hierarchy.GetComponent<Image> ();
         #endregion
     }
     
     // Update is called once per frame
     void Update () {
         raycast_crosshair();
         if (is_Interactive == true) {
             if (Input.GetKey ("e") == true) {
                 pick_up_object ();
             }
             change_crosshair_colour ();
         } else {
             change_crosshair_colour_to_default ();
         }
     }
 
     void raycast_crosshair () {
         if (Physics.Raycast (raycast_camera.transform.position, raycast_camera.transform.forward, out crosshair_hit, crosshair_range)) { //shoots a raycast
             if (crosshair_hit.transform.tag == "Interactive") { //if hit, checks if object is interactive
                 is_Interactive = true;
             } 
         } else {
             is_Interactive = false;
         }
     }
 
     void change_crosshair_colour () {
             cross_hair_img_component.color = highlight_colour;
     }
     void change_crosshair_colour_to_default () {
         cross_hair_img_component.color = default_colour;
 
     }
 
     void pick_up_object () {
         crosshair_hit.transform.position = raycast_camera.transform.position + raycast_camera.transform.forward * object_distance; //object in front of camera
         crosshair_hit.transform.rotation = new Quaternion( 0.0f, raycast_camera.transform.rotation.y, 0.0f, raycast_camera.transform.rotation.w ); //object rotates with camera
     }
 }